Задание выполнено

mongo-sharding-repl/README.md

@@ -0,0 +1,20 @@
+# Комментарии по обновленному compose.yaml:
+
+## Шардирование
+
+configsrv – конфигурационный сервер, который управляет метаданными о шардировании. Использует порт 27017.
+
+shard1 и shard2 – два шардированных набора для распределения данных, настроенные на порты 27018 и 27019 соответственно.
+
+mongos – маршрутизатор, работающий на порту 27020, через который будут проходить запросы к MongoDB. Настройки pymongo_api 
+перенаправлены на этот порт.
+
+pymongo_api – микросервис приложения, который теперь обращается к MongoDB через mongos для работы с шардами.
+
+## Репликация
+
+В каждом шарде настроена репликация с одной основной и двумя вторичными нодами.
+
+shard1-primary, shard1-secondary1, shard1-secondary2 — инстансы для первого шарда
+
+shard2-primary, shard2-secondary1, shard2-secondary2 — инстансы для второго шарда

mongo-sharding-repl/api_app/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.12.1-slim
+WORKDIR /app
+EXPOSE 8080
+COPY requirements.txt ./
+# Устанавливаем зависимости python не пересобирая их
+RUN pip install --no-cache --no-cache-dir -r requirements.txt
+# Копирование кода приложения
+COPY app.py /app/
+ENTRYPOINT ["uvicorn"]
+CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"]

mongo-sharding-repl/api_app/app.py

@@ -0,0 +1,192 @@
+import json
+import logging
+import os
+import time
+from typing import List, Optional
+
+import motor.motor_asyncio
+from bson import ObjectId
+from fastapi import Body, FastAPI, HTTPException, status
+from fastapi_cache import FastAPICache
+from fastapi_cache.backends.redis import RedisBackend
+from fastapi_cache.decorator import cache
+from logmiddleware import RouterLoggingMiddleware, logging_config
+from pydantic import BaseModel, ConfigDict, EmailStr, Field
+from pydantic.functional_validators import BeforeValidator
+from pymongo import errors
+from redis import asyncio as aioredis
+from typing_extensions import Annotated
+
+# Configure JSON logging
+logging.config.dictConfig(logging_config)
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+app.add_middleware(
+    RouterLoggingMiddleware,
+    logger=logger,
+)
+
+DATABASE_URL = os.environ["MONGODB_URL"]
+DATABASE_NAME = os.environ["MONGODB_DATABASE_NAME"]
+REDIS_URL = os.getenv("REDIS_URL", None)
+
+
+def nocache(*args, **kwargs):
+    def decorator(func):
+        return func
+
+    return decorator
+
+
+if REDIS_URL:
+    cache = cache
+else:
+    cache = nocache
+
+
+client = motor.motor_asyncio.AsyncIOMotorClient(DATABASE_URL)
+db = client[DATABASE_NAME]
+
+# Represents an ObjectId field in the database.
+# It will be represented as a `str` on the model so that it can be serialized to JSON.
+PyObjectId = Annotated[str, BeforeValidator(str)]
+
+
+@app.on_event("startup")
+async def startup():
+    if REDIS_URL:
+        redis = aioredis.from_url(REDIS_URL, encoding="utf8", decode_responses=True)
+        FastAPICache.init(RedisBackend(redis), prefix="api:cache")
+
+
+class UserModel(BaseModel):
+    """
+    Container for a single user record.
+    """
+
+    id: Optional[PyObjectId] = Field(alias="_id", default=None)
+    age: int = Field(...)
+    name: str = Field(...)
+
+
+class UserCollection(BaseModel):
+    """
+    A container holding a list of `UserModel` instances.
+    """
+
+    users: List[UserModel]
+
+
+@app.get("/")
+async def root():
+    collection_names = await db.list_collection_names()
+    collections = {}
+    for collection_name in collection_names:
+        collection = db.get_collection(collection_name)
+        collections[collection_name] = {
+            "documents_count": await collection.count_documents({})
+        }
+    try:
+        replica_status = await client.admin.command("replSetGetStatus")
+        replica_status = json.dumps(replica_status, indent=2, default=str)
+    except errors.OperationFailure:
+        replica_status = "No Replicas"
+
+    topology_description = client.topology_description
+    read_preference = client.client_options.read_preference
+    topology_type = topology_description.topology_type_name
+    replicaset_name = topology_description.replica_set_name
+
+    shards = None
+    if topology_type == "Sharded":
+        shards_list = await client.admin.command("listShards")
+        shards = {}
+        for shard in shards_list.get("shards", {}):
+            shards[shard["_id"]] = shard["host"]
+
+    cache_enabled = False
+    if REDIS_URL:
+        cache_enabled = FastAPICache.get_enable()
+
+    return {
+        "mongo_topology_type": topology_type,
+        "mongo_replicaset_name": replicaset_name,
+        "mongo_db": DATABASE_NAME,
+        "read_preference": str(read_preference),
+        "mongo_nodes": client.nodes,
+        "mongo_primary_host": client.primary,
+        "mongo_secondary_hosts": client.secondaries,
+        "mongo_address": client.address,
+        "mongo_is_primary": client.is_primary,
+        "mongo_is_mongos": client.is_mongos,
+        "collections": collections,
+        "shards": shards,
+        "cache_enabled": cache_enabled,
+        "status": "OK",
+    }
+
+
+@app.get("/{collection_name}/count")
+async def collection_count(collection_name: str):
+    collection = db.get_collection(collection_name)
+    items_count = await collection.count_documents({})
+    # status = await client.admin.command('replSetGetStatus')
+    # import ipdb; ipdb.set_trace()
+    return {"status": "OK", "mongo_db": DATABASE_NAME, "items_count": items_count}
+
+
+@app.get(
+    "/{collection_name}/users",
+    response_description="List all users",
+    response_model=UserCollection,
+    response_model_by_alias=False,
+)
+@cache(expire=60 * 1)
+async def list_users(collection_name: str):
+    """
+    List all of the user data in the database.
+    The response is unpaginated and limited to 1000 results.
+    """
+    time.sleep(1)
+    collection = db.get_collection(collection_name)
+    return UserCollection(users=await collection.find().to_list(1000))
+
+
+@app.get(
+    "/{collection_name}/users/{name}",
+    response_description="Get a single user",
+    response_model=UserModel,
+    response_model_by_alias=False,
+)
+async def show_user(collection_name: str, name: str):
+    """
+    Get the record for a specific user, looked up by `name`.
+    """
+
+    collection = db.get_collection(collection_name)
+    if (user := await collection.find_one({"name": name})) is not None:
+        return user
+
+    raise HTTPException(status_code=404, detail=f"User {name} not found")
+
+
+@app.post(
+    "/{collection_name}/users",
+    response_description="Add new user",
+    response_model=UserModel,
+    status_code=status.HTTP_201_CREATED,
+    response_model_by_alias=False,
+)
+async def create_user(collection_name: str, user: UserModel = Body(...)):
+    """
+    Insert a new user record.
+
+    A unique `id` will be created and provided in the response.
+    """
+    collection = db.get_collection(collection_name)
+    new_user = await collection.insert_one(
+        user.model_dump(by_alias=True, exclude=["id"])
+    )
+    created_user = await collection.find_one({"_id": new_user.inserted_id})
+    return created_user

mongo-sharding-repl/api_app/requirements.txt

@@ -0,0 +1,6 @@
+fastapi==0.110.2
+uvicorn[standard]==0.29.0
+motor==3.5.3
+redis==4.4.2
+fastapi-cache2==0.2.0
+logmiddleware==0.0.4

mongo-sharding-repl/compose.yaml

@@ -0,0 +1,98 @@
+services:
+  configsrv:
+    container_name: configsrv
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --configsvr --replSet configReplSet --port 27017
+    volumes:
+      - configsrv_data:/data/db
+    networks:
+      - mongo-network
+
+  shard1-primary:
+    container_name: shard1-primary
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard1ReplSet --port 27018
+    volumes:
+      - shard1_primary_data:/data/db
+    networks:
+      - mongo-network
+
+  shard1-secondary1:
+    container_name: shard1-secondary1
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard1ReplSet --port 27018
+    volumes:
+      - shard1_secondary1_data:/data/db
+    networks:
+      - mongo-network
+
+  shard1-secondary2:
+    container_name: shard1-secondary2
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard1ReplSet --port 27018
+    volumes:
+      - shard1_secondary2_data:/data/db
+    networks:
+      - mongo-network
+
+  shard2-primary:
+    container_name: shard2-primary
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard2ReplSet --port 27019
+    volumes:
+      - shard2_primary_data:/data/db
+    networks:
+      - mongo-network
+
+  shard2-secondary1:
+    container_name: shard2-secondary1
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard2ReplSet --port 27019
+    volumes:
+      - shard2_secondary1_data:/data/db
+    networks:
+      - mongo-network
+
+  shard2-secondary2:
+    container_name: shard2-secondary2
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongod --shardsvr --replSet shard2ReplSet --port 27019
+    volumes:
+      - shard2_secondary2_data:/data/db
+    networks:
+      - mongo-network
+
+  mongos:
+    container_name: mongos
+    image: dh-mirror.gitverse.ru/mongo:latest
+    command: mongos --configdb configReplSet/configsrv:27017 --port 27020
+    ports:
+      - 27020:27020
+    networks:
+      - mongo-network
+
+  pymongo_api:
+    container_name: pymongo_api
+    build: 
+      context: api_app
+      dockerfile: Dockerfile
+    image: kazhem/pymongo_api:1.0.0
+    depends_on:
+      - mongos
+    ports:
+      - 8080:8080
+    environment:
+      MONGODB_URL: "mongodb://mongos:27020"
+      MONGODB_DATABASE_NAME: "somedb"
+
+volumes:
+  configsrv_data:
+  shard1_primary_data:
+  shard1_secondary1_data:
+  shard1_secondary2_data:
+  shard2_primary_data:
+  shard2_secondary1_data:
+  shard2_secondary2_data:
+
+networks:
+  mongo-network:

mongo-sharding-repl/scripts/mongo-init.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+###
+# Инициализируем бд
+###
+
+docker compose exec -T mongodb1 mongosh <<EOF
+use somedb
+for(var i = 0; i < 1000; i++) db.helloDoc.insertOne({age:i, name:"ly"+i})
+EOF
+

mongo-sharding/README.md

@@ -0,0 +1,12 @@
+# Комментарии по обновленному compose.yaml:
+
+## Шардирование
+
+configsrv – конфигурационный сервер, который управляет метаданными о шардировании. Использует порт 27017.
+
+shard1 и shard2 – два шардированных набора для распределения данных, настроенные на порты 27018 и 27019 соответственно.
+
+mongos – маршрутизатор, работающий на порту 27020, через который будут проходить запросы к MongoDB. Настройки pymongo_api 
+перенаправлены на этот порт.
+
+pymongo_api – микросервис приложения, который теперь обращается к MongoDB через mongos для работы с шардами.

mongo-sharding/api_app/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.12.1-slim
+WORKDIR /app
+EXPOSE 8080
+COPY requirements.txt ./
+# Устанавливаем зависимости python не пересобирая их
+RUN pip install --no-cache --no-cache-dir -r requirements.txt
+# Копирование кода приложения
+COPY app.py /app/
+ENTRYPOINT ["uvicorn"]
+CMD ["app:app", "--host", "0.0.0.0", "--port", "8080"]