Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(docker-compose): namespace corpora services #59

Merged
merged 12 commits into from
Nov 20, 2024
2 changes: 1 addition & 1 deletion .corpora.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: skyl/corpora
server:
base_url: http://app:8877
base_url: http://corpora-app:8877
url: https://github.com/skyl/corpora
1 change: 0 additions & 1 deletion .devcontainer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ corpora plan issue

## Ports
- **App:** 8877
- **Database:** 5432

## Notes
- Variations in Docker setup can be found in `../docker-compose.yaml` and `../docker/README.md`.
Expand Down
5 changes: 2 additions & 3 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "Corpora DevContainer",
"dockerComposeFile": "../docker-compose.yaml",
"service": "interactive",
"service": "corpora-interactive",
"workspaceFolder": "/workspace",
"customizations": {
"vscode": {
Expand Down Expand Up @@ -47,8 +47,7 @@
],
"postCreateCommand": "/workspace/.devcontainer/setup.sh",
"forwardPorts": [
8877,
5432
8877
],
"remoteUser": "vscode"
}
37 changes: 26 additions & 11 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -1,24 +1,26 @@
services:
app:
corpora-app:
build:
context: .
dockerfile: docker/Dockerfile.app
volumes:
- .:/workspace
environment:
PYTHONPATH: "/workspace/py/packages"
REDIS_URL: "redis://redis:6379/0"
REDIS_URL: "redis://corpora-redis:6379/0"
OPENAI_API_KEY: "${OPENAI_API_KEY}"
command: python manage.py runserver 0.0.0.0:8877
working_dir: /workspace/py/packages/corpora_proj
depends_on:
- db
- redis
- celery
- corpora-db
- corpora-redis
- corpora-celery
ports:
- "8877:8877"
networks:
- shared-network

celery:
corpora-celery:
build:
context: .
dockerfile: docker/Dockerfile.app
Expand All @@ -27,13 +29,15 @@ services:
- .:/workspace
environment:
PYTHONPATH: "/workspace/py/packages"
REDIS_URL: "redis://redis:6379/0"
REDIS_URL: "redis://corpora-redis:6379/0"
OPENAI_API_KEY: "${OPENAI_API_KEY}"
depends_on:
redis:
corpora-redis:
condition: service_healthy
networks:
- shared-network

db:
corpora-db:
build:
context: .
dockerfile: docker/Dockerfile.pgvector
Expand All @@ -45,8 +49,10 @@ services:
- "5432:5432"
volumes:
- postgres-data:/var/lib/postgresql/data
networks:
- shared-network

redis:
corpora-redis:
image: redis:latest
ports:
- "6379:6379"
Expand All @@ -55,8 +61,10 @@ services:
interval: 5s
timeout: 5s
retries: 5
networks:
- shared-network

interactive:
corpora-interactive:
build:
context: .
dockerfile: docker/Dockerfile
Expand All @@ -69,6 +77,13 @@ services:
GITHUB_TOKEN: "${GITHUB_TOKEN}"
command: sleep infinity
working_dir: /workspace
networks:
- shared-network

volumes:
postgres-data:


networks:
shared-network:
name: shared-network
176 changes: 0 additions & 176 deletions md/notes/docker-compose-celery.md

This file was deleted.

7 changes: 0 additions & 7 deletions openapitools.json

This file was deleted.

2 changes: 1 addition & 1 deletion py/genall.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ set -e
rm -rf packages/corpora_client
mkdir -p packages/corpora_client/docs packages/corpora_client/test
rm -rf gen/corpora_client
openapi-generator-cli generate -i http://app:8877/api/openapi.json \
openapi-generator-cli generate -i http://corpora-app:8877/api/openapi.json \
-g python -o gen/corpora_client \
--additional-properties=packageName=corpora_client
# --additional-properties=library=asyncio
Expand Down
2 changes: 1 addition & 1 deletion py/openapitools.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
"generator-cli": {
"version": "7.10.0"
}
}
}
2 changes: 1 addition & 1 deletion py/packages/corpora/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

class BinaryView(View):
def get(self, request, arch, *args, **kwargs):
path = f"/workspace/rs/target/{arch}/release/corpora_cli"
path = f"/workspace/rs/target/{arch}/release/corpora"
if not os.path.exists(path):
return HttpResponse("Binary not found", status=404)

Expand Down
2 changes: 1 addition & 1 deletion py/packages/corpora_cli/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ class AuthError(Exception):
class AuthResolver:
def __init__(self, config: Dict[str, Any]):
self.config = config
base_url = config.get("base_url", "http://app:8877")
base_url = config.get("base_url", "http://corpora-app:8877")
self.token_url = f"{base_url}/o/token/"

def resolve_auth(self) -> str:
Expand Down
4 changes: 3 additions & 1 deletion py/packages/corpora_cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,9 @@ def get_api_clients(
# Configure and return the authenticated API client
client_config = corpora_client.Configuration()
# TODO: deploy and default to the main production server?
client_config.host = config.get("server", {}).get("base_url", "http://app:8877")
client_config.host = config.get("server", {}).get(
"base_url", "http://corpora-app:8877"
)
client_config.access_token = auth_token
return (
corpora_client.CorpusApi(corpora_client.ApiClient(client_config)),
Expand Down
10 changes: 5 additions & 5 deletions py/packages/corpora_cli/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def yaml_content():
"""Sample YAML content for testing."""
return """
server:
base_url: "http://app:8877"
base_url: "http://corpora-app:8877"
auth:
client_id: "${CORPORA_CLIENT_ID}"
client_secret: "${CORPORA_CLIENT_SECRET}"
Expand Down Expand Up @@ -44,7 +44,7 @@ def test_load_config_success(mock_yaml_load, mock_file_open, yaml_content, mocke
"""Test load_config with successful loading and env substitution."""
# Set up mock for yaml.safe_load to return the parsed YAML structure
mock_yaml_load.return_value = {
"server": {"base_url": "http://app:8877"},
"server": {"base_url": "http://corpora-app:8877"},
"auth": {
"client_id": "${CORPORA_CLIENT_ID}",
"client_secret": "${CORPORA_CLIENT_SECRET}",
Expand All @@ -60,7 +60,7 @@ def test_load_config_success(mock_yaml_load, mock_file_open, yaml_content, mocke
assert config["auth"]["client_id"] == "test_client_id"
assert config["auth"]["client_secret"] == "test_client_secret"
assert config["auth"]["optional_auth"] == "optional_auth_value"
assert config["server"]["base_url"] == "http://app:8877"
assert config["server"]["base_url"] == "http://corpora-app:8877"


@patch("builtins.open", new_callable=mock_open)
Expand Down Expand Up @@ -104,7 +104,7 @@ def test_substitute_env_variables(mocked_env):
"client_secret": "${CORPORA_CLIENT_SECRET}",
"nonexistent_var": "${NON_EXISTENT_VAR}",
},
"server": {"base_url": "http://app:8877"},
"server": {"base_url": "http://corpora-app:8877"},
}
substituted_config = substitute_env_variables(config)

Expand All @@ -113,4 +113,4 @@ def test_substitute_env_variables(mocked_env):
assert (
substituted_config["auth"]["nonexistent_var"] == ""
) # Non-existent vars default to ""
assert substituted_config["server"]["base_url"] == "http://app:8877"
assert substituted_config["server"]["base_url"] == "http://corpora-app:8877"
Loading
Loading