docker-compose.yml

version: "3.5"
name: lakefs-multimodal-data-demo-local
services:
  jupyter-notebook:
    build: .
    user: "1000:100"
    environment:
      # log-level is set to WARN because of noisy stdout problem
      # -> See https://github.com/jupyter-server/jupyter_server/issues/1279
      - NOTEBOOK_ARGS=--log-level='WARN' --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=/home/jovyan/notebooks
    ports:
      - 8893:8888 # Jupyter
      - 4043:4040 # Spark
      - 5002:5000 # MLflow
    volumes:
      - $PWD:/home/jovyan/notebooks
      - ../../data:/data
    entrypoint: ["/bin/bash", "-c"]
    command:
      - |
        echo 'Starting MLflow server in background'
        mlflow server -h 0.0.0.0 &
        echo 'Starting Jupyter'
        exec /usr/local/bin/start-notebook.sh

  lakefs:
    image: treeverse/lakefs-enterprise:latest
    pull_policy: always
    ports:
      - "8003:8000"
    depends_on:
      postgres:
        condition: service_healthy
      minio-setup:
        condition: service_completed_successfully
    environment:
      - LAKEFS_BLOCKSTORE_TYPE=s3
      - LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true
      - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://minio:9000
      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=minioadmin
      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=minioadmin
      - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
      - LAKEFS_LOGGING_LEVEL=INFO
      - LAKEFS_STATS_ENABLED=${LAKEFS_STATS_ENABLED:-1}
      - LAKECTL_CREDENTIALS_ACCESS_KEY_ID=AKIAIOSFOLKFSSAMPLES
      - LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
      - LAKECTL_SERVER_ENDPOINT_URL=http://localhost:8000
      - LAKEFS_DATABASE_TYPE=postgres
      - LAKEFS_DATABASE_POSTGRES_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
      - LAKEFS_AUTH_UI_CONFIG_RBAC=internal
      - LAKEFS_FEATURES_ICEBERG_CATALOG=true
      - LAKEFS_LICENSE_PATH=/tmp/lakefs/$LAKEFS_LICENSE_FILE_NAME
      - LAKEFS_INSTALLATION_FIXED_ID=$LAKEFS_INSTALLATION_ID
    volumes:
      - $PWD:/tmp/lakefs
    entrypoint: ["/bin/sh", "-c"]
    healthcheck:
      test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8000/api/v1/healthcheck"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 40s
    command:
        - |
          lakefs setup --user-name everything-bagel --access-key-id "$$LAKECTL_CREDENTIALS_ACCESS_KEY_ID" --secret-access-key "$$LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY" || true
          lakefs run &
          echo "---- Creating repository ----"
          wait-for -t 60 lakefs:8000 -- lakectl repo create lakefs://quickstart s3://quickstart --sample-data || true
          echo ""
          wait-for -t 60 minio:9000 && echo '------------------------------------------------

                MinIO admin:   http://127.0.0.1:9003/
                
                               Username : minioadmin
                               Password : minioadmin
                '
          echo "------------------------------------------------"
          wait-for -t 60 jupyter-notebook:8888 && echo '

                Jupyter:       http://127.0.0.1:8893/
                '
          echo "------------------------------------------------"
          echo ""
          echo "lakeFS Web UI: http://127.0.0.1:8003/      >(.＿.)<"
          echo "                                             (  )_ "
          echo ""
          echo "                Access Key ID    : $$LAKECTL_CREDENTIALS_ACCESS_KEY_ID"
          echo "                Secret Access Key: $$LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY"
          echo ""
          echo "-------- Let's go and have axolotl fun! --------"
          echo ""
          wait

  lakefs-init:
    image: treeverse/lakefs-enterprise:latest
    depends_on:
      lakefs:
        condition: service_healthy
    environment:
      LAKECTL_SERVER_ENDPOINT_URL: http://lakefs:8000
      LAKECTL_CREDENTIALS_ACCESS_KEY_ID: AKIAIOSFOLKFSSAMPLES
      LAKECTL_CREDENTIALS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
    entrypoint: ["/bin/sh", "-c"]
    command: >
      "echo 'starting LakeFS initialization';
      lakectl repo create lakefs://multimodal-data-local-repo s3://example/multimodal-data-local-repo --default-branch main || echo 'repo creation failed or already exists';
      echo 'LakeFS repository initialization completed';"

  lakefs-mds:
    image: treeverse/mds:latest
    ports:
      - "8004:8000"
    depends_on:
      lakefs-init:
        condition: service_completed_successfully
    volumes:
      - $PWD/mds-config.yaml:/app/config.yaml
    environment:
      AWS_ACCESS_KEY_ID: minioadmin
      AWS_SECRET_ACCESS_KEY: minioadmin
    restart: unless-stopped  # Restart if it crashes, but not if manually stopped
    stop_grace_period: 30s   # Wait 30 seconds for graceful shutdown before SIGKILL

  minio-setup:
    image: minio/mc:RELEASE.2023-05-18T16-59-00Z
    environment:
        - MC_HOST_lakefs=http://minioadmin:minioadmin@minio:9000
    depends_on:
      - minio
    volumes:
      - ../../data:/data
    entrypoint: ["/bin/sh", "-c"]
    command:
        - |
          mc mb lakefs/quickstart lakefs/example lakefs/sample-data
          mc cp --recursive /data/* lakefs/sample-data 1>/dev/null # don't be so noisy 🤫

  minio:
    image: minio/minio:RELEASE.2023-05-18T00-05-36Z
    ports:
      - "9002:9000"
      - "9003:9001"
    entrypoint: ["minio", "server", "/data", "--console-address", ":9001"]

  postgres:
    image: postgres:14
    ports:
      - "5434:5432"
    environment:
      POSTGRES_USER: lakefs
      POSTGRES_PASSWORD: lakefs
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "lakefs"]
      interval: 10s
      retries: 5
      start_period: 5s

networks:
  default:
    name: lakefs-multimodal-data-demo-local-network