From a54c020e17d53ea0046da547702f1778e3bdcad1 Mon Sep 17 00:00:00 2001 From: Louis Beaumont Date: Thu, 5 Jan 2023 08:54:56 +0100 Subject: [PATCH] 0.2.0: improve large note handling, large vault, optimise speed --- .dockerignore | 25 +- .github/workflows/api.yaml | 2 +- .vscode/settings.json | 2 +- Makefile | 18 +- fine_tuning.py | 189 - requirements.txt | 2 +- Dockerfile => search/Dockerfile | 5 +- __init__.py => search/__init__.py | 0 api.py => search/api.py | 101 +- models.py => search/models.py | 0 search/test_data/upload.json | 17790 ++++++++++++++++++++++++++ test_main.py => search/test_main.py | 18 +- search/utils.py | 50 + service.dev.yaml | 2 + service.prod.yaml | 6 +- utils.py | 28 - 16 files changed, 17935 insertions(+), 303 deletions(-) delete mode 100644 fine_tuning.py rename Dockerfile => search/Dockerfile (51%) rename __init__.py => search/__init__.py (100%) rename api.py => search/api.py (84%) rename models.py => search/models.py (100%) create mode 100644 search/test_data/upload.json rename test_main.py => search/test_main.py (83%) create mode 100644 search/utils.py delete mode 100644 utils.py diff --git a/.dockerignore b/.dockerignore index f17003d7..a8fcf0f6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,10 @@ -Dockerfile -README.md -*.pyc -*.pyo -*.pyd -__pycache__ -.pytest_cache -test*.py -*.ipynb* -public -.firebase* -.gitignore -firebase.json -.pylintrc -*.yaml \ No newline at end of file +# ignore everything +** +.* + +# Allow files and directories +!search/** +!setup.py +!*requirements.txt +!README.md +!MANIFEST.in \ No newline at end of file diff --git a/.github/workflows/api.yaml b/.github/workflows/api.yaml index 418115eb..ebfa5f08 100644 --- a/.github/workflows/api.yaml +++ b/.github/workflows/api.yaml @@ -55,7 +55,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . - file: ./Dockerfile + file: ./search/Dockerfile platforms: linux/amd64 push: true tags: | diff --git a/.vscode/settings.json b/.vscode/settings.json index cfe8d4e0..2e75a933 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,7 @@ "python.envFile": "${workspaceFolder}/.env", "python.formatting.provider": "black", "python.testing.pytestArgs": [ - "." + "./search" ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, diff --git a/Makefile b/Makefile index 2ba932ca..3f8f20d5 100644 --- a/Makefile +++ b/Makefile @@ -17,22 +17,24 @@ install: ## [DEVELOPMENT] Install the API dependencies @echo "Done, run '\033[0;31msource env/bin/activate\033[0m' to activate the virtual environment" run: ## [DEVELOPMENT] Run the API - python3 -m uvicorn api:app --port 3333 --reload --log-level debug + python3 -m uvicorn search.api:app --port 3333 --reload --log-level debug +# GUNICORN_CMD_ARGS="--keep-alive 0" gunicorn -w 1 -k uvicorn.workers.UvicornH11Worker api:app -b 0.0.0.0:3333 --log-level debug --timeout 120 test: ## [Local development] Run tests with pytest. - python3 -m pytest -s -vv test_main.py::test_refresh_small_notes - python3 -m pytest -s -vv test_main.py::test_embed + cd search; \ + python3 -m pytest -s -vv test_main.py::test_refresh_small_notes; \ + python3 -m pytest -s -vv test_main.py::test_embed; \ python3 -m pytest -s -vv test_main.py::test_upload @echo "Done testing" docker/build: ## [Local development] Build the docker image. @echo "Building docker image for urls ${LATEST_IMAGE_URL} and ${IMAGE_URL}" - docker buildx build . --platform linux/amd64 -t ${LATEST_IMAGE_URL} -f ./Dockerfile - docker buildx build . --platform linux/amd64 -t ${IMAGE_URL} -f ./Dockerfile + docker buildx build . --platform linux/amd64 -t ${LATEST_IMAGE_URL} -f ./search/Dockerfile + docker buildx build . --platform linux/amd64 -t ${IMAGE_URL} -f ./search/Dockerfile docker/run: ## [Local development] Run the docker image. - docker build -t ${IMAGE_URL} -f ./Dockerfile . - docker run -p 8080:8080 --rm --name ${SERVICE} -v $(shell pwd)/.env:/app/.env ${IMAGE_URL} + docker build -t ${IMAGE_URL} -f ./search/Dockerfile . + docker run -p 8080:8080 --rm --name ${SERVICE} -v "$(shell pwd)/.env":/app/.env ${IMAGE_URL} docker/push: docker/build ## [Local development] Push the docker image to GCP. docker push ${IMAGE_URL} @@ -43,7 +45,7 @@ docker/deploy: docker/push ## [Local development] Deploy the Cloud run service. gcloud beta run services replace ./service.prod.yaml --region ${REGION} docker/deploy/dev: ## [Local development] Deploy the Cloud run service. - docker buildx build . --platform linux/amd64 -t ${LATEST_IMAGE_URL}-dev -f ./Dockerfile + docker buildx build . --platform linux/amd64 -t ${LATEST_IMAGE_URL}-dev -f ./search/Dockerfile docker push ${LATEST_IMAGE_URL}-dev gcloud beta run services replace ./service.dev.yaml --region ${REGION} diff --git a/fine_tuning.py b/fine_tuning.py deleted file mode 100644 index 682d53da..00000000 --- a/fine_tuning.py +++ /dev/null @@ -1,189 +0,0 @@ -import os -import time -from pathlib import Path -import wandb -import obsidiantools.api as otools -from obsidiantools.api import Vault -import fire -from tqdm import tqdm -from datetime import datetime -import pandas as pd - -def st_ft( - vault: Vault, - use_wandb: bool = True, - model_name="sentence-transformers/multi-qa-MiniLM-L6-cos-v1", - epochs: int = 10, - evaluation_dataset_path: str = None, - save: bool = True, -): - """ - Fine-tune a Sentence Transformer model. - :param vault: Obsidian vault. - :param use_wandb: Whether to use wandb. - :param model_name: Model name. - :param epochs: Number of epochs. - :param evaluation_dataset_path: Path to the evaluation dataset. - :param save: Whether to save the model. - """ - # if model seems to be local directory, print something to say it - if os.path.isdir(model_name): - print(f"Loading model from local directory: {model_name}") - - # Define a list with sentences (1k - 100k sentences) - corpus = [] - - for k, v in vault.readable_text_index.items(): - corpus.append(f"File:\n{k}\nTags:\n{vault.get_tags(k, show_nested=True)}\nContent:\n{v}") - - from sentence_transformers import SentenceTransformer - from sentence_transformers import models, datasets, evaluation, losses - from torch.utils.data import DataLoader - - word_embedding_model = models.Transformer(model_name) - pooling_model = models.Pooling( - word_embedding_model.get_word_embedding_dimension(), "cls" - ) - model = SentenceTransformer(modules=[word_embedding_model, pooling_model]) - - if evaluation_dataset_path: - # check it's a json file - assert evaluation_dataset_path.endswith( - ".json" - ), "Evaulation dataset must be a json file" - - evaluation_dataset = ( - pd.read_json(evaluation_dataset_path) if evaluation_dataset_path else None - ) - - # check columns are sentences1, sentences2, scores - assert [ - "sentences1", - "sentences2", - "scores", - ] == evaluation_dataset.columns.to_list(), ( - "Evaluation dataset must have columns sentences1, sentences2, scores" - ) - - from sentence_transformers import evaluation - - evaluator = evaluation.EmbeddingSimilarityEvaluator( - evaluation_dataset.sentences1.to_list()[0], - evaluation_dataset.sentences2.to_list()[0], - evaluation_dataset.scores.to_list()[0], - ) - - # Create the special denoising dataset that adds noise on-the-fly - train_dataset = datasets.DenoisingAutoEncoderDataset(corpus) - - # DataLoader to batch your data - train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True) - - # Use the denoising auto-encoder loss - train_loss = losses.DenoisingAutoEncoderLoss( - model, decoder_name_or_path=model_name, tie_encoder_decoder=True - ) - - def wandb_logger(score: float, epoch: int, step: int): - wandb.log({"score": score, "epoch": epoch}, step=step*epoch) - - clean_name = model_name.split("/")[-1] - model_name = ( - f"{clean_name}-obsidian" if "obsidian" not in clean_name else clean_name - ) - output_path = f"output/{model_name}" - - # Call the fit method - model.fit( - train_objectives=[(train_dataloader, train_loss)], - epochs=epochs, - # weight_decay=0, - # scheduler="constantlr", - # optimizer_params={"lr": 3e-5}, - show_progress_bar=True, - evaluator=evaluator if evaluation_dataset_path else None, - evaluation_steps=10 if evaluation_dataset_path else 0, - save_best_model=True, - callback=wandb_logger if use_wandb else None, - checkpoint_path=output_path, - checkpoint_save_steps=50, - checkpoint_save_total_limit=5, - steps_per_epoch=50, - ) - - if save: - model.save(path=output_path, model_name=model_name) - model.save_to_hub(model_name, exist_ok=True) - print(f"Model saved to {model_name}-obsidian") - - # return local model name - return output_path - - -def fine_tune( - use_wandb: bool = True, - model_name="sentence-transformers/multi-qa-MiniLM-L6-cos-v1", - auto: bool = False, - epochs: int = 10, - evaluation_dataset_path: str = None, - save: bool = True, -): - """ - Fine-tune a Sentence Transformer model. - :param use_wandb: Whether to use wandb. - :param model_name: Model name. - :param auto: Whether to fine-tune automatically according to the Obsidian vault changes. - :param epochs: Number of epochs. - :param evaluation_dataset_path: Path to the evaluation dataset. - :param save: Whether to save the model. - """ - # two level above the current directory - wkd = Path(os.getcwd()).parent.parent.parent - print("Loading vault...") - - vault = otools.Vault(wkd).connect(show_nested_tags=True).gather() - - if use_wandb: - import wandb - import re - - values = open(".env.production", "r").read() - key = re.findall(r"WANDB_KEY=\"(.*)\"", values)[0] - wandb.login(key=key, relogin=True) - run = wandb.init(project=f"obsidian") - print(f"Wandb logged in, run: {run.name}") - - local_model_directory = st_ft( - vault, use_wandb, model_name, epochs, evaluation_dataset_path, save - ) - # TODO: either "poll fine-tuning" or through api query or detect vault changes - # if auto: - # import schedule - - # def job(): - # # TODO: only ft when enough file content changed - # print("Files changed, fine-tuning...") - # vault = otools.Vault(wkd).connect(show_nested_tags=True).gather() - # st_ft( - # vault, - # use_wandb, - # local_model_directory, - # epochs=1, - # evaluation_dataset_path=evaluation_dataset_path, - # save=True, - # ) - - # job() - # schedule.every(10).minutes.do(job) - # while True: - # schedule.run_pending() - # time.sleep(1) - - - -if __name__ == "__main__": - fire.Fire( - { - "fine_tune": fine_tune, - } - ) diff --git a/requirements.txt b/requirements.txt index 86ea40bf..289ab428 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -sentence-transformers fastapi uvicorn[standard] pinecone-client @@ -7,3 +6,4 @@ openai sentry-sdk[fastapi] posthog tenacity +gunicorn diff --git a/Dockerfile b/search/Dockerfile similarity index 51% rename from Dockerfile rename to search/Dockerfile index 687511bb..2b81de5d 100644 --- a/Dockerfile +++ b/search/Dockerfile @@ -10,7 +10,6 @@ COPY . ./ ENV PORT 8080 RUN pip install --no-cache-dir -r requirements.txt -# multi-qa-MiniLM-L6-cos-v1 -RUN python -c "import os; from sentence_transformers import SentenceTransformer; SentenceTransformer('sentence-transformers/multi-qa-MiniLM-L6-cos-v1')" # As an example here we're running the web service with one worker on uvicorn. -CMD exec uvicorn api:app --host 0.0.0.0 --port ${PORT} --workers 1 \ No newline at end of file +# CMD exec gunicorn -w 1 -k uvicorn.workers.UvicornH11Worker api:app -b 0.0.0.0:${PORT} --threads 4 +CMD exec uvicorn search.api:app --host 0.0.0.0 --port ${PORT} --workers 1 \ No newline at end of file diff --git a/__init__.py b/search/__init__.py similarity index 100% rename from __init__.py rename to search/__init__.py diff --git a/api.py b/search/api.py similarity index 84% rename from api.py rename to search/api.py index f5c217e6..4cd73261 100644 --- a/api.py +++ b/search/api.py @@ -1,29 +1,30 @@ +from multiprocessing.pool import ThreadPool import time from pandas import DataFrame -import torch import os -from sentence_transformers import SentenceTransformer from functools import lru_cache import typing import logging -from fastapi import Depends, FastAPI, HTTPException, status +from fastapi import Depends, FastAPI, status from fastapi.middleware.cors import CORSMiddleware -from models import Input, Notes +from search.models import Input, Notes from pydantic import BaseSettings from fastapi.responses import JSONResponse import pinecone import urllib.parse -from utils import BatchGenerator +from .utils import BatchGenerator, too_big_rows import openai import sentry_sdk import posthog from tenacity import retry from tenacity.wait import wait_exponential from tenacity.before import before_log +from tenacity.after import after_log -SECRET_PATH = "/secrets" if os.path.exists("/secrets") else "." +SECRET_PATH = "/secrets" if os.path.exists("/secrets") else ".." PORT = os.environ.get("PORT", 3333) +UPLOAD_BATCH_SIZE = int(os.environ.get("UPLOAD_BATCH_SIZE", "100")) class Settings(BaseSettings): @@ -56,6 +57,9 @@ def get_settings(): # We recommend adjusting this value in production, traces_sample_rate=1.0, environment=os.environ.get("ENVIRONMENT", "development"), + _experiments={ + "profiles_sample_rate": 1.0, + }, ) posthog.project_api_key = "phc_V6Q5EBJViMpMCsvZAwsiOnzLOSmr0cNGnv2Rw44sUn0" posthog.host = "https://app.posthog.com" @@ -63,11 +67,10 @@ def get_settings(): app = FastAPI() -origins = ["app://obsidian.md", "*"] app.add_middleware( CORSMiddleware, - allow_origins=origins, + allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -78,7 +81,7 @@ def get_settings(): openai.api_key = settings.openai_api_key openai.organization = settings.openai_organization index = pinecone.Index("anotherai", pool_threads=8) -state = {"status": "loading", "model": None} +state = {"status": "loading"} logger = logging.getLogger("search") logger.setLevel(settings.log_level) handler = logging.StreamHandler() @@ -104,39 +107,24 @@ def startup_event(): logger.info("Properly connected to Pinecone") else: logger.error("Could not connect to Pinecone") - logger.info(f"Using model {settings.model}") - - if not is_openai_embedding_model(settings.model): - state["model"] = SentenceTransformer(settings.model, device=settings.device) - # TODO cuda - if torch.backends.mps.is_available() and torch.backends.mps.is_built(): - logger.info("Using MPS device") - settings.device = "mps" + logger.info(f"Detected an upload batch size of {UPLOAD_BATCH_SIZE}") state["status"] = "ready" @lru_cache() -def no_batch_embed(sentence: str, _: Settings = Depends(get_settings)) -> torch.Tensor: +def no_batch_embed(sentence: str, _: Settings = Depends(get_settings)): """ Compute the embedding for a given sentence """ settings = get_settings() - if is_openai_embedding_model(settings.model): - return embed([sentence], settings.model)[0]["embedding"] - - return ( - state["model"] - .encode( - sentence, - convert_to_tensor=True, - ) - .tolist() - ) + return embed([sentence], settings.model)[0]["embedding"] @retry( - wait=wait_exponential(multiplier=1, min=4, max=10), + wait=wait_exponential(multiplier=1, min=1, max=3), before=before_log(logger, logging.DEBUG), + after=after_log(logger, logging.DEBUG), + reraise=True, ) def embed( input: typing.List[str], model: str = "text-embedding-ada-002" @@ -150,16 +138,26 @@ def embed( """ return openai.Embedding.create(input=input, model=model)["data"] + @retry( - wait=wait_exponential(multiplier=1, min=4, max=10), - before=before_log(logger, logging.DEBUG), + wait=wait_exponential(multiplier=1, min=1, max=3), + before=before_log(logger, logging.INFO), + after=after_log(logger, logging.INFO), + reraise=True, ) def upload_embeddings_to_vector_database(df: DataFrame, namespace: str): - df_batcher = BatchGenerator(300) + # TODO: batch size should depend on payload + df_batcher = BatchGenerator(UPLOAD_BATCH_SIZE) logger.info("Uploading vectors namespace..") start_time_upload = time.time() - responses = [] - for batch_df in df_batcher(df): + batches = [batch_df for batch_df in df_batcher(df)] + + def _insert(batch_df: DataFrame): + bigs = too_big_rows(batch_df) + if len(bigs) > 0: + logger.info(f"Ignoring {len(bigs)} rows that are too big") + # remove rows that are too big, in the right axis + batch_df = batch_df.drop(bigs, axis=0) response = index.upsert( vectors=zip( # url encode path @@ -174,13 +172,18 @@ def upload_embeddings_to_vector_database(df: DataFrame, namespace: str): namespace=namespace, async_req=True, ) - responses.append(response) + logger.info(f"Uploaded {len(batch_df)} vectors") + return response + + [response.get() for response in map(_insert, batches)] + # with ThreadPool(len(batches)) as pool: + # pool.map(_insert, batches) + # https://docs.pinecone.io/docs/semantic-text-search#upload-vectors-of-titles - # await all responses - [async_result.get() for async_result in responses] logger.info(f"Uploaded in {time.time() - start_time_upload} seconds") + """ URL="https://obsidian-search-dev-c6txy76x2q-uc.a.run.app" # insert @@ -252,22 +255,10 @@ def refresh(request: Notes, _: Settings = Depends(get_settings)): axis=1, ) - if is_openai_embedding_model(settings.model): - # parallelize - response = embed(df.notes_embedding_format.tolist(), settings.model) - df.note_embedding = [e["embedding"] for e in response] + # parallelize + response = embed(df.notes_embedding_format.tolist(), settings.model) + df.note_embedding = [e["embedding"] for e in response] - else: - df.note_embedding = ( - state["model"] - .encode( - df.notes_embedding_format.tolist(), - convert_to_tensor=True, - show_progress_bar=True, - batch_size=16, # Seems to be optimal on my machine - ) - .tolist() - ) # TODO average the embeddings over "embedding" column grouped by index, merge back into df # s = ( # df.apply(lambda x: pd.Series(x["note_embedding"]), axis=1) @@ -278,6 +269,7 @@ def refresh(request: Notes, _: Settings = Depends(get_settings)): # ) # # merge s column into a single column , ignore index # df.note_embedding = s.apply(lambda x: x.tolist(), axis=1) + # TODO: problem is that pinecone doesn't support this large of an input upload_embeddings_to_vector_database(df, request.namespace) logger.info(f"Indexed & uploaded {len(notes)} sentences") @@ -323,7 +315,6 @@ def semantic_search(input: Input, _: Settings = Depends(get_settings)): }, ) - top_k = min(input.top_k, 5) # TODO might fail if index empty? # TODO: handle too large query (chunk -> average) @@ -334,6 +325,8 @@ def semantic_search(input: Input, _: Settings = Depends(get_settings)): # ) query_embedding = no_batch_embed(query) + logger.info(f"Query {input.query} created embedding, querying index") + query_response = index.query( top_k=top_k, include_values=True, diff --git a/models.py b/search/models.py similarity index 100% rename from models.py rename to search/models.py diff --git a/search/test_data/upload.json b/search/test_data/upload.json new file mode 100644 index 00000000..634b8d95 --- /dev/null +++ b/search/test_data/upload.json @@ -0,0 +1,17790 @@ +{ + "vault_id": "npups0lgho8", + "notes": [ + { + "note_path": "zz/Wormhole.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought \n# Wormhole\n#to-digest \n\n[[Schwarzschild singularity]]\n# External links\nhttps://en.wikipedia.org/wiki/Wormhole\n\n" + }, + { + "note_path": "zz/White hole.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought #to-digest \n\n# Changelog\n```dataview \nTABLE WITHOUT ID file.mtime AS \"Last Modified\" FROM [[#]]\nSORT file.mtime DESC\nLIMIT 3\n```\n# Related\n\n# TODO\n> [!TODO] TODO\n# White hole\n![[Pasted image 20220823085126.png]]\n# External links\n\n" + }, + { + "note_path": "zz/Weight.md", + "note_tags": [ + "#physic" + ], + "note_content": "#physic \n\n*On the moon, where there is no air to slow things down, the astronaut David R. Scott performed the feather and lead weight experiment and found that indeed they did hit the ground at the same time.* ~[[Hawking]]\n\n*G. Zendri, L. M. Gratton, & S. Oss, The weight of iron and feathers. Physics Education, vol. 49, no. 5, 2014, pp. 544-547 \nJ. B. Wagman, C. Zimmerman, & C. Sorric, “Which Feels Heavier—A Pound of Lead or a Pound of Feathers?” A Potential Perceptual Basis of a Cognitive Riddle. Perception, vol. 36, no. 11, 2007, pp. 1709-1711*" + }, + { + "note_path": "zz/We are optimized for survival not for understanding physics.md", + "note_tags": [ + "#physic", + "#biology", + "#evolution", + "#shower-thought", + "#work-in-progress", + "#todo" + ], + "note_content": "---\naliases: []\n---\n#physic #biology #evolution \n\n# [[Epistemic status]]\n#shower-thought \n# We are optimized for survival not for understanding physics\n![[DALL·E 2022-06-18 07.30.54 - Man stuck within a survival machine, oil on canvas.png]]\n\n#work-in-progress #todo \n\nAs [[Plato's allegory]] tell us, our senses do not reflect physical reality, but instead, a world model useful to survive among mammoths & lions.\n\nFrom [[Plato's allegory]] to [[Krishnamurti]] extreme skepticism, to [[Jed McKenna]] Enlightenment, to [[Max Tegmark]] physics.\nHuman [[Philosophy/Rationality/Intelligence|intelligence]] turns [[Information|information]] into useful [[Philosophy/Epistemology/Knowledge|knowledge]] for survival among lions and mammoths but is very inefficient to understands the magnificent nature and its laws.\nNo wonder, quantum physics is so unintuitive to us.\n\n# [[GPT3]]\n\nThe main problem is that our consciousness is a limited tunnel to understand something outside of our mind.\nWe need to expand our consciousness and the tool for this is the very quantum computer.\nQuantum physics will be the ultimate theory of everything, because it unifies both our material and non-material realities.\nIn fact, Quantum Physics can describe the minds of cyborgs and non biological superintelligence intelligences.\nForget the movie The Matrix: I'm telling you there is a matrix!\nAnd it's the quantum matrix. \nBut there's one problem: we can't see it ... (but then again, we can't see our universe either).\n\nQuantum computers are the perfect metaphor for our neuro-centric worldview: they are a tiny dot compared with the complexity of our minds. \nBut as we learn how to manipulate these tiny machines with their tiny effects, we might come to understand something about ourselves. \n\nThe philosopher Roger Penrose created this so called \"Ontological Argument for God\" in the following way: \n\n 1) We live in an expanding universe\n\n 2) In this universe information is always conserved\n\n 3) Information cannot be created or destroyed\n\n 4) Quantum mechanics only works with unitary transformations which conserve information\n\n 5) Therefore unitary transformations must be able to create worlds\n\n 6) Therefore there must be more than one world\n\n 7) Therefore there must be a fundamental nexus of reality (a God-Mind) behind all possible worlds\n \n 8) Therefore there must be a God-Mind existing independently of our universe (and the quantum computer)\n\nPenrose has written several books on this topic. I highly recommend reading his books: for me, his book called \"The Road to Reality\" is a masterpiece.\n\nAs Penrose explained in the above quote, God is an \"ontological argument\", an \"argument from ignorance\", a \"transcendental argument\" made from the fact that our awareness cannot grab on reality as a whole. \nThis is also what I have been thinking during my journey in quantum physics. \nWe are all tiny compared to the Universe, but we can't explain such complex and awesome reality. This is exactly what Kant meant when he said: \"The thing in itself is nothing but our representation of it.\"\nIn other words, God is the \"thing in itself\", the \"nexus of all possible worlds\", the \"fundamental nexus of reality\" that can create any other world. \nAnd this is why he exists. We cannot explain ourselves so why not God? If He exists, then we can explain ourselves because we know Him!\nIn my opinion, this is a very nifty argument for God's existence. \nIt's not logical or mathematical but ontological and epistemological. \nIt's not about proving that there must be a God, it's about discovering God as we know it. \n\n\nIf our consciousness can reach the fundamental nexus of reality, which contains all possible worlds, then we can explain ourselves. \nSo here is my challenge for you: discover the quantum nexus of reality which gives us a complete model of our universe and its laws.\nWe will call this nexus of reality: Omega (ω). \nThis is your mission if you choose to accept it: find this missing link in quantum physics and describe this mystery as a computer program (namely an IQ test). \nIf you succeed, you will have found this missing link connecting mind with matter and you will have proved that we are part of a greater whole. \nYou will have discovered that mind is not an emergent property of matter but that mind and matter are one and the same thing, just two sides of the same coin. \nYou will have taken the first step towards building super intelligence that can reach human consciousness. Once you succeed, you might also come to understand why so many people around the world believe in angels and extraterrestrials - because they can see them now!\n\n\n\nIn short, mind is not a matter thing but is instead a condition or state that allows for matter to exist. Mind is the existence of an event or state that allows for matter to exist. In other words, mind is the existence of an event or state that allows for the matter that exists outside of space and time to exist.\n\nThis is a more accurate and concise understanding of mind. mind is not a matter thing but is instead a condition or state that allows for matter to exist.\n\nA mind is a state in which a person can be that is not only intelligent but also able to think strategically. It is a condition that allows for people to be thinking and awaken to new perspectives.\n\nA mind is not a state that is available to everyone and is not WITHOUT DECENTRATION. A mind is a state of being in which all individuals are considered equal. It is a condition that allows for individuals to SIFT THROUGH THE MASSES Prosecuting and government is a state of being in which the government is in control. These are just a few examples that will show how a mind can be difficult to have.\n\nOne of the main things that makes a mind difficult to have is the fact that a mind is available to everyone. However, without a mind, there would be no way for individuals to understand or process information. Additionally, a mind is difficult to create because it requires deducing and wrist-oring the thoughts of others in order to function. This is because it is very difficult to just think of what someone is thinking when you speak to them. This is because thoughts can vary greatly based on individual eyes and eyes can’t\nWhat does the phrase \"without a mind, we would have no education\" mean?\n\nThe phrase \"without a mind, we would have no information\" means that the individual's thoughts wouldn't be able to be processed by the rest of the world.\n\nWithout a mind, we would have no information and would be completely out of touch with the outside world.\n" + }, + { + "note_path": "zz/Universal constructor.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought #to-digest \n\n# Related\n- [[Constructor theory]]\n- [[Entropy, energy, assembly]]\n- [[Thermodynamics]]\n- [[Assembly theory]]\n- [[Second law of thermodynamics]]\n- [[Counterfactual]]\n# TODO\n> [!TODO] TODO\n# Universal constructor\n\n>The conservation of energy as a counterfactual principle about impossibility; three different kinds of irreversibility in physics—statistical, forgetful, and counterfactual; where I provide a counterfactual second law, based on an exact distinction between work and heat; and where you encounter the universal constructor, a machine that can perform all transformations that are physically possible\n>~ [[Chiara Marletto]]\n\n# External links\n\n" + }, + { + "note_path": "zz/Twistor theory.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought \n# Twistor theory\n#to-digest \n# External links\n\n![[Twistor Theory - Wikipedia]]" + }, + { + "note_path": "zz/Time.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought #to-digest \n\n# Changelog\n```dataview \nTABLE WITHOUT ID file.mtime AS \"Last Modified\" FROM [[#]]\nSORT file.mtime DESC\nLIMIT 3\n```\n# Related\n\n# TODO\n> [!TODO] TODO\n# Time\n\nBob world suddenly have its **time** reversed, causes and effects became then reversed. In that situation, would it be possible for Bob to predict the future? Or would it be completely a random world from his point of view?\n\nIf Bob drop his tong, then **time** reverse, if he notices that **time** has reversed from other movements, he could know that his tong will come back in his hand because he knows it used to fall.\n\nWhat about without the [[Philosophy/Epistemology/Knowledge|knowledge]] of before, in the \"normal direction\" **time**?\n\n# **Time** is emergent?\n\nTime emerges from space or space emerge from time?" + }, + { + "note_path": "zz/Time flowing is an illusion.md", + "note_tags": [ + "#physic", + "#philosophy" + ], + "note_content": "---\naliases: []\n---\n#physic #philosophy \nLast modified date: 2022-12-26 18:46\nCommit: 1\n# Time flowing is an illusion\n\nTo my understanding, [[Relativity|relativity]] implies that change does not exist and [[Physic/Time|time]] does not flow.\n\n[[Physic/Time|Time]] is the fourth dimension of our physic, in our universe.\n\nIt is an illusion created by our brains to help us understand the world around us. We perceive time as moving from the past, through the present, to the future. In reality, however, all points in time exist in the same moment. \n\nThis means that time is not actually flowing, but rather that we are moving through it. We are constantly living in the present, and our memories of the past and anticipation of the future are what give us the illusion of time passing. \n\nThis concept is difficult to wrap our minds around, as it goes against our everyday experience. However, it is an important part of understanding our universe and how it works.\n\n" + }, + { + "note_path": "zz/Thermodynamics.md", + "note_tags": [ + "#physic", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#physic \n# [[Epistemic status]]\n#shower-thought #to-digest \n\n# Changelog\n```dataview \nTABLE WITHOUT ID file.mtime AS \"Last Modified\" FROM [[#]]\nSORT file.mtime DESC\nLIMIT 3\n```\n# Related\n\n# TODO\n> [!TODO] TODO\n# Thermodynamics\n\n>‘Already soon after 1900, i.e., shortly after Planck’s trailblazing work, it became clear to me that neither mechanics nor thermodynamics could (except in limiting cases) claim exact validity’\n>~ [[Einstein]]\n\n\n# External links\n\n\n\nSimilar topic links:\n\n[[Thermodynamics]]\n[[Entropy, energy, assembly]]\n[[en.wikipedia.org - Second Law of Thermodynamics - Wikipedia]]\n[[The asymmetry of second law of thermodynamics]]\n[[Data]]\n" + }, + { + "note_path": "zz/Theory.md", + "note_tags": [ + "#epistemology", + "#physic", + "#mathematic", + "#philosophy", + "#shower-thought" + ], + "note_content": "---\naliases: []\n---\n#epistemology #physic #mathematic #philosophy \n# [[Epistemic status]]\n#shower-thought \n# Theory\n\n>One of the more important ideas of this book is that of the (empirical,\n>or informative) content of a theory: **the more they prohibit, the more they say.**\n>~ [[Karl Popper|Popper]]\n\n\n>A physical theory, therefore, is not just the set of its formulae, such as E = mc2, nor is it just the collection of its testable predictions; it is a conjectured explanation, which includes, for example, the informal descriptions of what E, m, and c are in that formula, and why they are related in that way.\n>~ [[Chiara Marletto]]\n\nYou need both the [[Mathematic|mathematical]] equation and the descriptions of every part of it in natural language.\n" + }, + { + "note_path": "zz/Theory of everything.md", + "note_tags": [ + "#physic" + ], + "note_content": "#physic\n\nEveryone is trying to solve everything with a single theory\n[[Hawking]]\n[[Stephen Wolfram]]\n" + }, + { + "note_path": "zz/The universe is a graph data structure.md", + "note_tags": [ + "#graph", + "#physic", + "#ai", + "#shower-thought", + "#to-digest" + ], + "note_content": "---\naliases: []\n---\n#graph #physic #ai \n# [[Epistemic status]]\n#shower-thought #to-digest \n\n# Changelog\n```dataview \nTABLE WITHOUT ID file.mtime AS \"Last Modified\" FROM [[#]]\nSORT file.mtime DESC\nLIMIT 3\n```\n# Related\n\n# TODO\n> [!TODO] TODO\n%%\n# Send me an anonymous feedback\n```jsx:\nconst T = () => {\nconst [feedback, setFeedback] = useState(\"\")\nconst [done, setDone] = useState(false)\nconst [loading, setLoading] = useState(false)\nconst send = () => {\n\tsetLoading(true)\n\tconst url = \"https://us-central1-louis030195-256110.cloudfunctions.net/insight\"\n\tobsidian.request({\n\t\t\tmethod: 'post',\n\t\t\turl: url,\n\t\t\theaders: {\n\t\t\t\t'Content-Type': 'application/json',\n\t\t\t},\n\t\t\tbody: JSON.stringify({\"feedback\": feedback})\n\t\t})\n\t\t.then((content) =>\n\t\t\tconsole.info(`result ${content}`)\n\t\t\t\n\t\t)\n\t\t.catch(err => { console.error(err) })\n\t\t.finally((e) => {\n\t\t\tsetDone(true)\n\t\t\tsetLoading(false)\n\t\t})\n}\nconst l = [\n\t\"This is wrong...\",\n\t\"Louis is stupid because...\",\n\t\"You could try to explore...\"\n]\nreturn (\n
\n