From dda16e7bbaef40fa12c391abd8a204ce0cb0db2d Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 4 Jul 2023 17:33:10 +0100 Subject: [PATCH 01/25] adding postgres and migrations stuff as well as a small service that pushes data into the table --- .gitignore | 6 ++ README.md | 36 +++++++++ compose/docker-compose-local.yml | 40 +++++++--- .../20230704163029_InitialCreate.Designer.cs | 78 +++++++++++++++++++ .../20230704163029_InitialCreate.cs | 40 ++++++++++ .../RequestLoggerContextModelSnapshot.cs | 76 ++++++++++++++++++ src/Repository/Models/HttpRequest.cs | 54 +++++++++++++ src/Repository/Repository.csproj | 21 +++++ src/Repository/RequestLoggerContext.cs | 42 ++++++++++ .../RequestLoggerContextConfiguration.cs | 57 ++++++++++++++ src/RequestLogger.sln | 6 ++ .../Dto/Converters/RequestConverter.cs | 21 +++++ src/RequestLogger/Dto/Request.cs | 44 +++++++++++ src/RequestLogger/Program.cs | 37 ++++++++- src/RequestLogger/RequestLogger.csproj | 10 ++- .../Interfaces/IRequestLoggerService.cs | 8 ++ .../Services/RequestLoggerService.cs | 23 ++++++ .../Settings/RequestLoggerSettings.cs | 6 ++ .../appsettings.Development.json | 9 ++- 19 files changed, 598 insertions(+), 16 deletions(-) create mode 100644 src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs create mode 100644 src/Repository/Migrations/20230704163029_InitialCreate.cs create mode 100644 src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs create mode 100644 src/Repository/Models/HttpRequest.cs create mode 100644 src/Repository/Repository.csproj create mode 100644 src/Repository/RequestLoggerContext.cs create mode 100644 src/Repository/RequestLoggerContextConfiguration.cs create mode 100644 src/RequestLogger/Dto/Converters/RequestConverter.cs create mode 100644 src/RequestLogger/Dto/Request.cs create mode 100644 src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs create mode 100644 src/RequestLogger/Services/RequestLoggerService.cs create mode 100644 src/RequestLogger/Settings/RequestLoggerSettings.cs diff --git a/.gitignore b/.gitignore index 64fddec..c1f1852 100644 --- a/.gitignore +++ b/.gitignore @@ -397,3 +397,9 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml *.idea/ + +# Docker +*.env + +# Project specific +*/appsettings.Development.json \ No newline at end of file diff --git a/README.md b/README.md index d2ff2bd..7a86df2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,38 @@ # RequestLogger An API used to log all requests so that they can be reviewed at a later point + +### Local installation + +#### Docker + +This project makes use of entity framework code-first and a postgres SQL database. in order to run the application, the following steps must be followed. + +first there is a local docker compose file found in the `compose` directory. This can be run from that directory with the following command: + +```powershell +docker-compose -f docker-compose-local.yml up +``` +**Note:** the postgres container needs environment variables in a `.env` file. There's an example in the project under `.env-dist`, but it's reccomended to change the username and password to something more secure. + +This will run 4 containers. These are as follows: + +- [Nginx](https://hub.docker.com/_/nginx) + - The application is designed to run as a [mirror](http://nginx.org/en/docs/http/ngx_http_mirror_module.html) for the main application behind Nginx. This container is to mimic this behaviour +- [Mock Server](https://www.mock-server.com/#what-is-mockserver) + - This is to mock out a succesful response from all endpoints going through the initial `proxy_pass` from Nginx. it will always respond with `{"test": "test"}` +- [PostgreSQL](https://hub.docker.com/_/postgres) + - This is the database that holds all the request being logged by the request logger +- A built version of the Request Logger application + - This is built from the Dockerfile found in RequestLogger + +#### Entity Framework + +Once the docker containers are built, the database can be constructed by running the following command from the `src` directory + +**Note:** you will need to replace the `User Id` and `Password` values with your own + +```powershell +dotnet ef database update --project .\Repository\ --connection "Server=127.0.0.1;Port=5452;Database=postgres;User Id=;Password=;" +``` + +Alternatively, the `RunMigrations` value can be set to `true` in the `appsettings` and the migrations will be run when the code is run from the project. \ No newline at end of file diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 525b7f1..3136523 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -25,17 +25,35 @@ services: - nginx.docker mockserver: - image: mockserver/mockserver:latest - container_name: mock-server - environment: - MOCKSERVER_WATCH_INITIALIZATION_JSON: "true" - MOCKSERVER_PROPERTY_FILE: /config/mockserver.properties - MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializerJson.json - volumes: - - ../src/MockServerConfig/initializerJson.json:/config/initializerJson.json:ro - networks: - - nginx.docker + image: mockserver/mockserver:latest + container_name: mock-server + environment: + MOCKSERVER_WATCH_INITIALIZATION_JSON: "true" + MOCKSERVER_PROPERTY_FILE: /config/mockserver.properties + MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializerJson.json + volumes: + - ../src/MockServerConfig/initializerJson.json:/config/initializerJson.json:ro + networks: + - nginx.docker + + postgres: + image: postgres:12.15 + container_name: postgres + hostname: postgres + ports: + - "5452:5432" + volumes: + - rl_postgres_data:/var/lib/postgresql/data + - rl_postgres_data_backups:/backups + env_file: + - .env + networks: + - nginx.docker networks: nginx.docker: - name: nginx.docker \ No newline at end of file + name: nginx.docker + +volumes: + rl_postgres_data: {} + rl_postgres_data_backups: {} \ No newline at end of file diff --git a/src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs b/src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs new file mode 100644 index 0000000..489e481 --- /dev/null +++ b/src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs @@ -0,0 +1,78 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(RequestLoggerContext))] + [Migration("20230704163029_InitialCreate")] + partial class InitialCreate + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseSerialColumns(modelBuilder); + + modelBuilder.Entity("Repository.Models.HttpRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseSerialColumn(b.Property("Id")); + + b.Property("Body") + .HasColumnType("text") + .HasColumnName("body"); + + b.Property("Customer") + .HasColumnType("text") + .HasColumnName("customer"); + + b.Property("Headers") + .HasColumnType("text") + .HasColumnName("headers"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("QueryParams") + .HasColumnType("text") + .HasColumnName("query_params"); + + b.Property("RequestTime") + .HasColumnType("timestamp") + .HasColumnName("request_time"); + + b.Property("Service") + .HasColumnType("text") + .HasColumnName("service"); + + b.Property("Verb") + .IsRequired() + .HasColumnType("text") + .HasColumnName("verb"); + + b.HasKey("Id") + .HasName("pk_requests"); + + b.ToTable("requests", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Repository/Migrations/20230704163029_InitialCreate.cs b/src/Repository/Migrations/20230704163029_InitialCreate.cs new file mode 100644 index 0000000..d7152d5 --- /dev/null +++ b/src/Repository/Migrations/20230704163029_InitialCreate.cs @@ -0,0 +1,40 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace Repository.Migrations +{ + public partial class InitialCreate : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "requests", + columns: table => new + { + id = table.Column(type: "integer", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn), + verb = table.Column(type: "text", nullable: false), + service = table.Column(type: "text", nullable: true), + customer = table.Column(type: "text", nullable: true), + path = table.Column(type: "text", nullable: false), + query_params = table.Column(type: "text", nullable: true), + body = table.Column(type: "text", nullable: true), + headers = table.Column(type: "text", nullable: true), + request_time = table.Column(type: "timestamp", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_requests", x => x.id); + }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "requests"); + } + } +} diff --git a/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs b/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs new file mode 100644 index 0000000..eb586bd --- /dev/null +++ b/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs @@ -0,0 +1,76 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository; + +#nullable disable + +namespace Repository.Migrations +{ + [DbContext(typeof(RequestLoggerContext))] + partial class RequestLoggerContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "6.0.19") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseSerialColumns(modelBuilder); + + modelBuilder.Entity("Repository.Models.HttpRequest", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("integer") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseSerialColumn(b.Property("Id")); + + b.Property("Body") + .HasColumnType("text") + .HasColumnName("body"); + + b.Property("Customer") + .HasColumnType("text") + .HasColumnName("customer"); + + b.Property("Headers") + .HasColumnType("text") + .HasColumnName("headers"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text") + .HasColumnName("path"); + + b.Property("QueryParams") + .HasColumnType("text") + .HasColumnName("query_params"); + + b.Property("RequestTime") + .HasColumnType("timestamp") + .HasColumnName("request_time"); + + b.Property("Service") + .HasColumnType("text") + .HasColumnName("service"); + + b.Property("Verb") + .IsRequired() + .HasColumnType("text") + .HasColumnName("verb"); + + b.HasKey("Id") + .HasName("pk_requests"); + + b.ToTable("requests", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/Repository/Models/HttpRequest.cs b/src/Repository/Models/HttpRequest.cs new file mode 100644 index 0000000..fa29ff6 --- /dev/null +++ b/src/Repository/Models/HttpRequest.cs @@ -0,0 +1,54 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace Repository.Models; + +public class HttpRequest +{ + /// + /// Unique identifier for a request + /// + [Key] + public int Id { get; set; } + + /// + /// The http verb associated with the request + /// + public string Verb { get; set; } = null!; + + /// + /// The calling service + /// + public string? Service { get; set; } + + /// + /// The customer calling the service + /// + public string? Customer { get; set; } + + /// + /// The HTTP path of the request + /// + public string Path { get; set; } = null!; + + /// + /// The query parameters on the request + /// + public string? QueryParams { get; set; } + + /// + /// The body of the request + /// + public string? Body { get; set; } + + /// + /// The headers associated with a request + /// + public string? Headers { get; set; } + + /// + /// The time the request was made + /// + [Column(TypeName = "timestamp")] + public DateTime RequestTime { get; set; } +} \ No newline at end of file diff --git a/src/Repository/Repository.csproj b/src/Repository/Repository.csproj new file mode 100644 index 0000000..95f128e --- /dev/null +++ b/src/Repository/Repository.csproj @@ -0,0 +1,21 @@ + + + + net6.0 + enable + enable + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + diff --git a/src/Repository/RequestLoggerContext.cs b/src/Repository/RequestLoggerContext.cs new file mode 100644 index 0000000..bff77af --- /dev/null +++ b/src/Repository/RequestLoggerContext.cs @@ -0,0 +1,42 @@ +using Microsoft.EntityFrameworkCore; +using Repository.Models; +using Serilog; + +namespace Repository; + +public class RequestLoggerContext : DbContext +{ + /// + /// Context class for entity framework + /// + public RequestLoggerContext() + { + } + + /// + /// Context class for entity framework + /// + /// The db context options + /// The postgres database connection string + public RequestLoggerContext(DbContextOptions options) + : base(options) + { + } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (!optionsBuilder.IsConfigured) + { + optionsBuilder.UseNpgsql(); + } + + optionsBuilder.UseSnakeCaseNamingConvention(); + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.UseSerialColumns(); + } + + public virtual DbSet Requests { get; set; } = null!; +} \ No newline at end of file diff --git a/src/Repository/RequestLoggerContextConfiguration.cs b/src/Repository/RequestLoggerContextConfiguration.cs new file mode 100644 index 0000000..4ac4806 --- /dev/null +++ b/src/Repository/RequestLoggerContextConfiguration.cs @@ -0,0 +1,57 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Serilog; + +namespace Repository; + +public static class RequestLoggerContextConfiguration +{ + private static readonly string ConnectionStringKey = "PostgreSQLConnection"; + private static readonly string RunMigrationsKey = "RunMigrations"; + + /// + /// Register and configure + /// + public static IServiceCollection AddRequestLoggerContext(this IServiceCollection services, + IConfiguration configuration) + => services + .AddDbContext(options => SetupOptions(configuration, options)); + private static void SetupOptions(IConfiguration configuration, + DbContextOptionsBuilder optionsBuilder) + => optionsBuilder.UseNpgsql(configuration.GetConnectionString(ConnectionStringKey) ?? string.Empty); + + /// + /// Run EF migrations if "RunMigrations" = true + /// + public static void TryRunMigrations(IConfiguration configuration) + { + if (configuration.GetValue(RunMigrationsKey, false)) + { + using var context = new RequestLoggerContext(GetOptionsBuilder(configuration).Options); + + var pendingMigrations = context.Database.GetPendingMigrations().ToList(); + if (pendingMigrations.Count == 0) + { + Log.Information("No migrations to run"); + return; + } + + Log.Information("Running migrations: {Migrations}", string.Join(",", pendingMigrations)); + context.Database.Migrate(); + } + } + + /// + /// Get a new instantiated object + /// + public static RequestLoggerContext GetNewDbContext(IConfiguration configuration) + => new(GetOptionsBuilder(configuration).Options); + + private static DbContextOptionsBuilder GetOptionsBuilder(IConfiguration configuration) + { + var optionsBuilder = new DbContextOptionsBuilder(); + SetupOptions(configuration, optionsBuilder); + return optionsBuilder; + } +} \ No newline at end of file diff --git a/src/RequestLogger.sln b/src/RequestLogger.sln index 4b966d4..2dd04fc 100644 --- a/src/RequestLogger.sln +++ b/src/RequestLogger.sln @@ -2,6 +2,8 @@ Microsoft Visual Studio Solution File, Format Version 12.00 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RequestLogger", "RequestLogger\RequestLogger.csproj", "{84CD6D44-A4F7-4FBC-8961-0FB7BF2D2D35}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Repository", "Repository\Repository.csproj", "{760E0EFC-C21E-46BA-9A39-C6E1D72D012E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -12,5 +14,9 @@ Global {84CD6D44-A4F7-4FBC-8961-0FB7BF2D2D35}.Debug|Any CPU.Build.0 = Debug|Any CPU {84CD6D44-A4F7-4FBC-8961-0FB7BF2D2D35}.Release|Any CPU.ActiveCfg = Release|Any CPU {84CD6D44-A4F7-4FBC-8961-0FB7BF2D2D35}.Release|Any CPU.Build.0 = Release|Any CPU + {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection EndGlobal diff --git a/src/RequestLogger/Dto/Converters/RequestConverter.cs b/src/RequestLogger/Dto/Converters/RequestConverter.cs new file mode 100644 index 0000000..513fb52 --- /dev/null +++ b/src/RequestLogger/Dto/Converters/RequestConverter.cs @@ -0,0 +1,21 @@ +using HttpRequest = Repository.Models.HttpRequest; + +namespace RequestLogger.Dto.Converters; + +public static class RequestConverter +{ + public static HttpRequest ConvertRequest(Request request) + { + return new HttpRequest + { + Verb = request.Verb, + Service = request.Service, + Customer = request.Customer, + Path = request.Path, + QueryParams = request.QueryParams, + Body = request.Body, + Headers = request.Headers, + RequestTime = request.RequestTime + }; + } +} \ No newline at end of file diff --git a/src/RequestLogger/Dto/Request.cs b/src/RequestLogger/Dto/Request.cs new file mode 100644 index 0000000..d709d36 --- /dev/null +++ b/src/RequestLogger/Dto/Request.cs @@ -0,0 +1,44 @@ +namespace RequestLogger.Dto; + +public class Request +{ + /// + /// The http verb associated with the request + /// + public string Verb { get; set; } = null!; + + /// + /// The calling service + /// + public string? Service { get; set; } + + /// + /// The customer calling the service + /// + public string? Customer { get; set; } + + /// + /// The HTTP path of the request + /// + public string Path { get; set; } = null!; + + /// + /// The query parameters on the request + /// + public string? QueryParams { get; set; } + + /// + /// The body of the request + /// + public string? Body { get; set; } + + /// + /// The headers associated with a request + /// + public string? Headers { get; set; } + + /// + /// The time the request was made + /// + public DateTime RequestTime { get; set; } +} \ No newline at end of file diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 30fba39..d2617b5 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -1,9 +1,13 @@ +using System.Text.Json; +using Repository; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Services.Interfaces; +using RequestLogger.Settings; using Serilog; var builder = WebApplication.CreateBuilder(args); -// Add services to the container. - // remove default logging providers builder.Logging.ClearProviders(); // Serilog configuration @@ -14,11 +18,17 @@ // Register Serilog builder.Logging.AddSerilog(logger); +builder.Services.AddSingleton(); builder.Services.AddControllers(); // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); +builder.Services.Configure(builder.Configuration.GetSection("RequestLogger")); + +builder.Services.AddRequestLoggerContext(builder.Configuration); +builder.Services.AddScoped(); + var app = builder.Build(); // Configure the HTTP request pipeline. @@ -26,6 +36,7 @@ { app.UseSwagger(); app.UseSwaggerUI(); + RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); } app.UseHttpsRedirection(); @@ -38,8 +49,28 @@ context.Request.Body.Position = 0; var rawRequestBody = await new StreamReader(context.Request.Body).ReadToEndAsync(); - Console.WriteLine(rawRequestBody); + //Console.WriteLine(rawRequestBody); + + using var scoped = app.Services.CreateScope(); + var loggerService = scoped.ServiceProvider.GetRequiredService(); + var request = new Request() + { + Verb = context.Request.Method, + Service = context.Request.Scheme, + Customer = context.Request.Headers.Authorization, + Path = context.Request.Path, + QueryParams = context.Request.QueryString.ToString(), + Body = rawRequestBody, + Headers = JsonSerializer.Serialize(context.Request.Headers), + RequestTime = DateTime.Now + }; + + Console.WriteLine(JsonSerializer.Serialize(request)); + + await loggerService.WriteLogMessage(request); + + await context.Response.WriteAsync("Ok"); return; diff --git a/src/RequestLogger/RequestLogger.csproj b/src/RequestLogger/RequestLogger.csproj index 4061258..25c40d9 100644 --- a/src/RequestLogger/RequestLogger.csproj +++ b/src/RequestLogger/RequestLogger.csproj @@ -8,8 +8,12 @@ + + + + - + @@ -21,4 +25,8 @@ + + + + diff --git a/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs b/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs new file mode 100644 index 0000000..207af4e --- /dev/null +++ b/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs @@ -0,0 +1,8 @@ +using RequestLogger.Dto; + +namespace RequestLogger.Services.Interfaces; + +public interface IRequestLoggerService +{ + Task WriteLogMessage(Request request); +} \ No newline at end of file diff --git a/src/RequestLogger/Services/RequestLoggerService.cs b/src/RequestLogger/Services/RequestLoggerService.cs new file mode 100644 index 0000000..4cbcdc5 --- /dev/null +++ b/src/RequestLogger/Services/RequestLoggerService.cs @@ -0,0 +1,23 @@ +using Repository; +using RequestLogger.Dto; +using RequestLogger.Dto.Converters; +using RequestLogger.Services.Interfaces; + +namespace RequestLogger.Services; + +public class RequestLoggerService : IRequestLoggerService +{ + private RequestLoggerContext _context; + public RequestLoggerService(RequestLoggerContext context) + { + _context = context; + } + + public async Task WriteLogMessage(Request request) + { + var httpRequest = RequestConverter.ConvertRequest(request); + await _context.Requests.AddAsync(httpRequest); + + await _context.SaveChangesAsync(); + } +} \ No newline at end of file diff --git a/src/RequestLogger/Settings/RequestLoggerSettings.cs b/src/RequestLogger/Settings/RequestLoggerSettings.cs new file mode 100644 index 0000000..3affcbd --- /dev/null +++ b/src/RequestLogger/Settings/RequestLoggerSettings.cs @@ -0,0 +1,6 @@ +namespace RequestLogger.Settings; + +public class RequestLoggerSettings +{ + +} \ No newline at end of file diff --git a/src/RequestLogger/appsettings.Development.json b/src/RequestLogger/appsettings.Development.json index 0c208ae..974bff8 100644 --- a/src/RequestLogger/appsettings.Development.json +++ b/src/RequestLogger/appsettings.Development.json @@ -4,5 +4,12 @@ "Default": "Information", "Microsoft.AspNetCore": "Warning" } - } + }, + "RequestLogger" : { + + }, + "ConnectionStrings": { + "PostgreSQLConnection": "Server=127.0.0.1;Port=5452;Database=postgres;User Id=postgres;Password=rl_password;" + }, + "RunMigrations": false } From 73e8056c7ab9fd046102fb18cd05b88799ac50b2 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 6 Jul 2023 15:38:50 +0100 Subject: [PATCH 02/25] adding in test suite --- .gitignore | 3 +- README.md | 16 +- compose/.env-dist | 11 + compose/docker-compose-local.yml | 10 +- ... 20230705100711_InitialCreate.Designer.cs} | 14 +- ...ate.cs => 20230705100711_InitialCreate.cs} | 8 +- .../RequestLoggerContextModelSnapshot.cs | 12 +- src/Repository/Models/HttpRequest.cs | 7 +- src/Repository/RequestLoggerContext.cs | 25 +-- .../RequestLoggerContextConfiguration.cs | 5 +- .../RequestLogger.Tests.csproj | 34 +++ .../Unit/BlacklistServiceTests.cs | 141 +++++++++++++ src/RequestLogger.Tests/Unit/ProgramTests.cs | 199 ++++++++++++++++++ .../Unit/RequestLoggerServiceTests.cs | 64 ++++++ src/RequestLogger.Tests/Usings.cs | 1 + src/RequestLogger.sln | 11 + .../Controllers/WeatherForecastController.cs | 32 --- .../Dto/Converters/RequestConverter.cs | 7 +- src/RequestLogger/Dto/Request.cs | 20 +- src/RequestLogger/Program.cs | 79 +++++-- src/RequestLogger/RequestLogger.csproj | 6 + .../Services/BlacklistService.cs | 53 +++++ .../Services/Interfaces/IBlacklistService.cs | 8 + .../Interfaces/IRequestLoggerService.cs | 2 +- .../Services/RequestLoggerService.cs | 21 +- .../Settings/BlacklistSettings.cs | 12 ++ .../Settings/RequestLoggerSettings.cs | 2 +- src/RequestLogger/WeatherForecast.cs | 12 -- src/nginx.conf | 1 + 29 files changed, 690 insertions(+), 126 deletions(-) create mode 100644 compose/.env-dist rename src/Repository/Migrations/{20230704163029_InitialCreate.Designer.cs => 20230705100711_InitialCreate.Designer.cs} (81%) rename src/Repository/Migrations/{20230704163029_InitialCreate.cs => 20230705100711_InitialCreate.cs} (83%) create mode 100644 src/RequestLogger.Tests/RequestLogger.Tests.csproj create mode 100644 src/RequestLogger.Tests/Unit/BlacklistServiceTests.cs create mode 100644 src/RequestLogger.Tests/Unit/ProgramTests.cs create mode 100644 src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs create mode 100644 src/RequestLogger.Tests/Usings.cs delete mode 100644 src/RequestLogger/Controllers/WeatherForecastController.cs create mode 100644 src/RequestLogger/Services/BlacklistService.cs create mode 100644 src/RequestLogger/Services/Interfaces/IBlacklistService.cs create mode 100644 src/RequestLogger/Settings/BlacklistSettings.cs delete mode 100644 src/RequestLogger/WeatherForecast.cs diff --git a/.gitignore b/.gitignore index c1f1852..361fe35 100644 --- a/.gitignore +++ b/.gitignore @@ -402,4 +402,5 @@ FodyWeavers.xsd *.env # Project specific -*/appsettings.Development.json \ No newline at end of file +*/appsettings.Development.json +*/appsettings.Docker.json \ No newline at end of file diff --git a/README.md b/README.md index 7a86df2..8798c61 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,18 @@ This will run 4 containers. These are as follows: - A built version of the Request Logger application - This is built from the Dockerfile found in RequestLogger +#### RequestLogger Settings + +by default, the application is set to use settings from the `appsettings.Docker.json` app settings file. If changes are made to this file after the containers are built, the command `docker-compose -f docker-compose-local.yml build` will need to be run from the `compose` folder. + #### Entity Framework -Once the docker containers are built, the database can be constructed by running the following command from the `src` directory +Once the docker containers are built, the database can be constructed by setting the `RunMigrations` value to `true` in the `appsettings` and the migrations will be run on startup. -**Note:** you will need to replace the `User Id` and `Password` values with your own +##### Adding migrations -```powershell -dotnet ef database update --project .\Repository\ --connection "Server=127.0.0.1;Port=5452;Database=postgres;User Id=;Password=;" -``` +Migrations can be added with the following commaand being run from the `src` directory -Alternatively, the `RunMigrations` value can be set to `true` in the `appsettings` and the migrations will be run when the code is run from the project. \ No newline at end of file +```powershell +dotnet ef migrations add -p .\Repository\ -s .\RequestLogger\ +``` \ No newline at end of file diff --git a/compose/.env-dist b/compose/.env-dist new file mode 100644 index 0000000..d16df0b --- /dev/null +++ b/compose/.env-dist @@ -0,0 +1,11 @@ +# PostgreSQL +# ------------------------------------------------------------------------------ +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=rl_password + +# Request Logger +# ------------------------------------------------------------------------------- +ASPNETCORE_ENVIRONMENT=Docker \ No newline at end of file diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 3136523..6a9ae46 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -23,6 +23,8 @@ services: dockerfile: RequestLogger\Dockerfile networks: - nginx.docker + env_file: + - .env mockserver: image: mockserver/mockserver:latest @@ -41,12 +43,12 @@ services: container_name: postgres hostname: postgres ports: - - "5452:5432" + - "5452:5432" volumes: - - rl_postgres_data:/var/lib/postgresql/data - - rl_postgres_data_backups:/backups + - rl_postgres_data:/var/lib/postgresql/data + - rl_postgres_data_backups:/backups env_file: - - .env + - .env networks: - nginx.docker diff --git a/src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs b/src/Repository/Migrations/20230705100711_InitialCreate.Designer.cs similarity index 81% rename from src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs rename to src/Repository/Migrations/20230705100711_InitialCreate.Designer.cs index 489e481..acbc367 100644 --- a/src/Repository/Migrations/20230704163029_InitialCreate.Designer.cs +++ b/src/Repository/Migrations/20230705100711_InitialCreate.Designer.cs @@ -12,7 +12,7 @@ namespace Repository.Migrations { [DbContext(typeof(RequestLoggerContext))] - [Migration("20230704163029_InitialCreate")] + [Migration("20230705100711_InitialCreate")] partial class InitialCreate { protected override void BuildTargetModel(ModelBuilder modelBuilder) @@ -22,7 +22,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasAnnotation("ProductVersion", "6.0.19") .HasAnnotation("Relational:MaxIdentifierLength", 63); - NpgsqlModelBuilderExtensions.UseSerialColumns(modelBuilder); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); modelBuilder.Entity("Repository.Models.HttpRequest", b => { @@ -31,10 +31,10 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("id"); - NpgsqlPropertyBuilderExtensions.UseSerialColumn(b.Property("Id")); + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); b.Property("Body") - .HasColumnType("text") + .HasColumnType("jsonb") .HasColumnName("body"); b.Property("Customer") @@ -42,7 +42,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnName("customer"); b.Property("Headers") - .HasColumnType("text") + .HasColumnType("jsonb") .HasColumnName("headers"); b.Property("Path") @@ -55,7 +55,7 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasColumnName("query_params"); b.Property("RequestTime") - .HasColumnType("timestamp") + .HasColumnType("timestamp with time zone") .HasColumnName("request_time"); b.Property("Service") @@ -71,6 +71,8 @@ protected override void BuildTargetModel(ModelBuilder modelBuilder) .HasName("pk_requests"); b.ToTable("requests", (string)null); + + b.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); }); #pragma warning restore 612, 618 } diff --git a/src/Repository/Migrations/20230704163029_InitialCreate.cs b/src/Repository/Migrations/20230705100711_InitialCreate.cs similarity index 83% rename from src/Repository/Migrations/20230704163029_InitialCreate.cs rename to src/Repository/Migrations/20230705100711_InitialCreate.cs index d7152d5..345961f 100644 --- a/src/Repository/Migrations/20230704163029_InitialCreate.cs +++ b/src/Repository/Migrations/20230705100711_InitialCreate.cs @@ -15,15 +15,15 @@ protected override void Up(MigrationBuilder migrationBuilder) columns: table => new { id = table.Column(type: "integer", nullable: false) - .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.SerialColumn), + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), verb = table.Column(type: "text", nullable: false), service = table.Column(type: "text", nullable: true), customer = table.Column(type: "text", nullable: true), path = table.Column(type: "text", nullable: false), query_params = table.Column(type: "text", nullable: true), - body = table.Column(type: "text", nullable: true), - headers = table.Column(type: "text", nullable: true), - request_time = table.Column(type: "timestamp", nullable: false) + body = table.Column(type: "jsonb", nullable: true), + headers = table.Column(type: "jsonb", nullable: true), + request_time = table.Column(type: "timestamp with time zone", nullable: false) }, constraints: table => { diff --git a/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs b/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs index eb586bd..030c3e9 100644 --- a/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs +++ b/src/Repository/Migrations/RequestLoggerContextModelSnapshot.cs @@ -20,7 +20,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasAnnotation("ProductVersion", "6.0.19") .HasAnnotation("Relational:MaxIdentifierLength", 63); - NpgsqlModelBuilderExtensions.UseSerialColumns(modelBuilder); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); modelBuilder.Entity("Repository.Models.HttpRequest", b => { @@ -29,10 +29,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("integer") .HasColumnName("id"); - NpgsqlPropertyBuilderExtensions.UseSerialColumn(b.Property("Id")); + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); b.Property("Body") - .HasColumnType("text") + .HasColumnType("jsonb") .HasColumnName("body"); b.Property("Customer") @@ -40,7 +40,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("customer"); b.Property("Headers") - .HasColumnType("text") + .HasColumnType("jsonb") .HasColumnName("headers"); b.Property("Path") @@ -53,7 +53,7 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnName("query_params"); b.Property("RequestTime") - .HasColumnType("timestamp") + .HasColumnType("timestamp with time zone") .HasColumnName("request_time"); b.Property("Service") @@ -69,6 +69,8 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasName("pk_requests"); b.ToTable("requests", (string)null); + + b.HasAnnotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); }); #pragma warning restore 612, 618 } diff --git a/src/Repository/Models/HttpRequest.cs b/src/Repository/Models/HttpRequest.cs index fa29ff6..3dcd746 100644 --- a/src/Repository/Models/HttpRequest.cs +++ b/src/Repository/Models/HttpRequest.cs @@ -1,14 +1,10 @@ -using System.ComponentModel.DataAnnotations; -using System.ComponentModel.DataAnnotations.Schema; - -namespace Repository.Models; +namespace Repository.Models; public class HttpRequest { /// /// Unique identifier for a request /// - [Key] public int Id { get; set; } /// @@ -49,6 +45,5 @@ public class HttpRequest /// /// The time the request was made /// - [Column(TypeName = "timestamp")] public DateTime RequestTime { get; set; } } \ No newline at end of file diff --git a/src/Repository/RequestLoggerContext.cs b/src/Repository/RequestLoggerContext.cs index bff77af..cc9fa93 100644 --- a/src/Repository/RequestLoggerContext.cs +++ b/src/Repository/RequestLoggerContext.cs @@ -1,6 +1,7 @@ using Microsoft.EntityFrameworkCore; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; using Repository.Models; -using Serilog; + namespace Repository; @@ -17,25 +18,25 @@ public RequestLoggerContext() /// Context class for entity framework /// /// The db context options - /// The postgres database connection string - public RequestLoggerContext(DbContextOptions options) + public RequestLoggerContext(DbContextOptions options) : base(options) { } - + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { - if (!optionsBuilder.IsConfigured) - { - optionsBuilder.UseNpgsql(); - } - - optionsBuilder.UseSnakeCaseNamingConvention(); } - + protected override void OnModelCreating(ModelBuilder modelBuilder) { - modelBuilder.UseSerialColumns(); + modelBuilder.Entity(builder => + { + builder.HasKey(i => i.Id); + builder.HasAnnotation("Npgsql:ValueGenerationStrategy", + NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + builder.Property(p => p.Body).HasColumnType("jsonb"); + builder.Property(p => p.Headers).HasColumnType("jsonb"); + }); } public virtual DbSet Requests { get; set; } = null!; diff --git a/src/Repository/RequestLoggerContextConfiguration.cs b/src/Repository/RequestLoggerContextConfiguration.cs index 4ac4806..89b5df5 100644 --- a/src/Repository/RequestLoggerContextConfiguration.cs +++ b/src/Repository/RequestLoggerContextConfiguration.cs @@ -19,7 +19,9 @@ public static IServiceCollection AddRequestLoggerContext(this IServiceCollection .AddDbContext(options => SetupOptions(configuration, options)); private static void SetupOptions(IConfiguration configuration, DbContextOptionsBuilder optionsBuilder) - => optionsBuilder.UseNpgsql(configuration.GetConnectionString(ConnectionStringKey) ?? string.Empty); + => optionsBuilder + .UseNpgsql(configuration.GetConnectionString(ConnectionStringKey) ?? string.Empty) + .UseSnakeCaseNamingConvention(); /// /// Run EF migrations if "RunMigrations" = true @@ -29,7 +31,6 @@ public static void TryRunMigrations(IConfiguration configuration) if (configuration.GetValue(RunMigrationsKey, false)) { using var context = new RequestLoggerContext(GetOptionsBuilder(configuration).Options); - var pendingMigrations = context.Database.GetPendingMigrations().ToList(); if (pendingMigrations.Count == 0) { diff --git a/src/RequestLogger.Tests/RequestLogger.Tests.csproj b/src/RequestLogger.Tests/RequestLogger.Tests.csproj new file mode 100644 index 0000000..4a24e52 --- /dev/null +++ b/src/RequestLogger.Tests/RequestLogger.Tests.csproj @@ -0,0 +1,34 @@ + + + + net6.0 + enable + enable + + false + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + diff --git a/src/RequestLogger.Tests/Unit/BlacklistServiceTests.cs b/src/RequestLogger.Tests/Unit/BlacklistServiceTests.cs new file mode 100644 index 0000000..9a5935d --- /dev/null +++ b/src/RequestLogger.Tests/Unit/BlacklistServiceTests.cs @@ -0,0 +1,141 @@ +using AutoFixture; +using FluentAssertions; +using Microsoft.Extensions.Options; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Settings; + +namespace RequestLogger.Tests.Unit; + +public class BlacklistServiceTests +{ + private readonly BlacklistService _blacklistService; + private readonly Fixture _fixture; + + public BlacklistServiceTests() + { + var settings = new RequestLoggerSettings() + { + BlacklistSettings = new BlacklistSettings() + { + EndpointBlacklist = new List() + { + "/blacklist/path" + }, + HeaderBlacklist = new List() + { + "blacklist" + }, + BodyStorageBlacklist = new List() + { + "/blacklist/body" + }, + QueryParamBlacklist = new List() + { + "blacklist" + } + } + }; + + _blacklistService = new BlacklistService(Options.Create(settings)); + _fixture = new Fixture(); + } + + [Fact] + public void CheckBlacklist_ReturnsUnmodifiedObject_WhenCalledCorrectly() + { + // Arrange + var request = _fixture.Create(); + + // Act + var checkedBlacklist = _blacklistService.CheckBlacklist(request); + + //Assert + checkedBlacklist.DoNotStore.Should().BeFalse(); + checkedBlacklist.request?.Body.Should().Be(request.Body); + checkedBlacklist.request?.Headers.Should().Contain(request.Headers); + checkedBlacklist.request?.Path.Should().Be(request.Path); + checkedBlacklist.request?.QueryParams.Should().Contain(request.QueryParams); + } + + [Fact] + public void CheckBlacklist_ReturnsDoNotStoreAsTrue_WhenCalledWithBlacklistedPath() + { + // Arrange + var request = _fixture. + Build() + .With(r => r.Path, "/blacklist/path") + .Create(); + + // Act + var checkedBlacklist = _blacklistService.CheckBlacklist(request); + + //Assert + checkedBlacklist.DoNotStore.Should().BeTrue(); + checkedBlacklist.request.Should().BeNull(); + } + + [Fact] + public void CheckBlacklist_ReturnsWithoutHeader_WhenCalledWithBlacklistedHeader() + { + // Arrange + var headers = new Dictionary { { "blacklist", "whoCares" } }; + + var request = _fixture. + Build() + .With(r => r.Headers, headers) + .Create(); + + // Act + var checkedBlacklist = _blacklistService.CheckBlacklist(request); + + //Assert + checkedBlacklist.DoNotStore.Should().BeFalse(); + checkedBlacklist.request?.Body.Should().Be(request.Body); + checkedBlacklist.request?.Headers.Count.Should().Be(0); + checkedBlacklist.request?.Path.Should().Be(request.Path); + checkedBlacklist.request?.QueryParams.Should().Contain(request.QueryParams); + } + + [Fact] + public void CheckBlacklist_ReturnsWithRemovedBody_WhenCalledWithBlacklistedBodyStoragePath() + { + // Arrange + var request = _fixture. + Build() + .With(r => r.Path, "/blacklist/body") + .Create(); + + // Act + var checkedBlacklist = _blacklistService.CheckBlacklist(request); + + //Assert + checkedBlacklist.DoNotStore.Should().BeFalse(); + checkedBlacklist.request?.Body.Should().Be("{ \"jsonRemoved\": \"json\" }"); + checkedBlacklist.request?.Headers.Should().Contain(request.Headers); + checkedBlacklist.request?.Path.Should().Be(request.Path); + checkedBlacklist.request?.QueryParams.Should().Contain(request.QueryParams); + } + + [Fact] + public void CheckBlacklist_ReturnsWithRemovedQueryParam_WhenCalledWithBlacklistedQueryParam() + { + // Arrange + var queryParams = new Dictionary { { "blacklist", "whoCares" } }; + + var request = _fixture. + Build() + .With(r => r.QueryParams, queryParams) + .Create(); + + // Act + var checkedBlacklist = _blacklistService.CheckBlacklist(request); + + //Assert + checkedBlacklist.DoNotStore.Should().BeFalse(); + checkedBlacklist.request?.Body.Should().Be(request.Body); + checkedBlacklist.request?.Headers.Should().Contain(request.Headers); + checkedBlacklist.request?.Path.Should().Be(request.Path); + checkedBlacklist.request?.QueryParams?.Count.Should().Be(0); + } +} \ No newline at end of file diff --git a/src/RequestLogger.Tests/Unit/ProgramTests.cs b/src/RequestLogger.Tests/Unit/ProgramTests.cs new file mode 100644 index 0000000..b7fbc01 --- /dev/null +++ b/src/RequestLogger.Tests/Unit/ProgramTests.cs @@ -0,0 +1,199 @@ +using System.Text.Json; +using FakeItEasy; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Repository; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Services.Interfaces; +using AutoFixture; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; + +#pragma warning disable CS8620 + +namespace RequestLogger.Tests.Unit; + +public class ProgramTests +{ + private readonly WebApplicationFactory _sut; + private readonly Fixture _fixture; + + public ProgramTests() + { + var root = new InMemoryDatabaseRoot(); + + _sut = new WebApplicationFactory() + .WithWebHostBuilder(builder => builder.ConfigureServices(services => + { + var descriptor = services.SingleOrDefault( + d => d.ServiceType == + typeof(DbContextOptions)); + + services.Remove(descriptor!); + services.AddScoped(); + services.AddScoped(); + services.AddDbContext(x => x.UseInMemoryDatabase("Testing", root)); + })); + + _fixture = new Fixture(); + } + + [Fact] + public void Program_CallingGet_ReturnsResponse() + { + // Arrange + var request = _fixture.Create(); + + var fakeRequestLoggerService = A.Fake(); + A.CallTo(() => fakeRequestLoggerService.WriteLogMessage(A.Ignored)).Returns(Task.FromResult(request)); + + var app = new WebApplicationFactory() + .WithWebHostBuilder(builder => builder.ConfigureServices(services => + { + services.AddScoped(_ => fakeRequestLoggerService); + })); + + var client = app.CreateClient(); + + // Act + var response = client.GetAsync("/stuff/").Result; + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + response.Content.ReadAsStringAsync().Result.Should().Contain(request.Verb); + } + + [Fact] + public void Program_MakingCall_ReturnsResponseFromDatabase() + { + // Arrange + var client = _sut.CreateClient(); + + + + // Act + var response = client.GetAsync("/stuff/").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingCall_ReturnsCustomerSuccessfully() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.GetAsync("/stuff/customer/3425234").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().Be("3425234"); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingCall_ReturnsQueryParamsSuccessfully() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.GetAsync("/some/url/3425234?query=something").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().ContainKey("query"); + } + + [Fact] + public void Program_MakingPostCall_ReturnsCorrectVerb() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.PostAsync("/some/uri", new StringContent("")).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingPostCallWithJsonBody_ReturnsCorrectBody() + { + // Arrange + var client = _sut.CreateClient(); + var jsonBody = "{\"test\": \"test\"}"; + + // Act + var response = client.PostAsync("/some/uri", new StringContent(jsonBody)).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().Be(jsonBody); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() + { + // Arrange + var client = _sut.CreateClient(); + var requestBody = "test"; + var jsonBody = "{ \"invalidJson\": \"test\" }"; + + // Act + var response = client.PostAsync("/some/uri", new StringContent(requestBody)).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().Be(jsonBody); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } +} \ No newline at end of file diff --git a/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs b/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs new file mode 100644 index 0000000..d1511f1 --- /dev/null +++ b/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs @@ -0,0 +1,64 @@ +using AutoFixture; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Options; +using Repository; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Settings; + +namespace RequestLogger.Tests.Unit; + +public class RequestLoggerServiceTests +{ + private readonly RequestLoggerService _requestLoggerService; + + private readonly RequestLoggerContext _requestLoggerContext; + + private readonly Fixture _fixture; + + public RequestLoggerServiceTests() + { + var root = new InMemoryDatabaseRoot(); + + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase("test", root).Options; + _requestLoggerContext = new RequestLoggerContext(options); + + _requestLoggerService = new RequestLoggerService(_requestLoggerContext, new BlacklistService(Options.Create(new RequestLoggerSettings()))); + _fixture = new Fixture(); + } + + [Fact] + public void WriteLogMessage_ReturnsLogMessage_WhenCalledCorrectly() + { + // Arrange + var request = _fixture.Create(); + + // Act + var response = _requestLoggerService.WriteLogMessage(request).Result; + + //Assert + response?.Body.Should().Be(request.Body); + response?.Verb.Should().Be(request.Verb); + response?.Customer.Should().Be(request.Customer); + } + + [Fact] + public void WriteLogMessage_ActuallyWritesToDatabase_WhenCalledCorrectly() + { + // Arrange + var request = _fixture.Create(); + + // Act + _ = _requestLoggerService.WriteLogMessage(request).Result; + + //Assert + var databaseItem = _requestLoggerContext.Requests.FirstOrDefault(r => r.Id == 1); + + databaseItem?.Body.Should().Be(request.Body); + databaseItem?.Verb.Should().Be(request.Verb); + databaseItem?.Customer.Should().Be(request.Customer); + } +} \ No newline at end of file diff --git a/src/RequestLogger.Tests/Usings.cs b/src/RequestLogger.Tests/Usings.cs new file mode 100644 index 0000000..8c927eb --- /dev/null +++ b/src/RequestLogger.Tests/Usings.cs @@ -0,0 +1 @@ +global using Xunit; \ No newline at end of file diff --git a/src/RequestLogger.sln b/src/RequestLogger.sln index 2dd04fc..3c76ffa 100644 --- a/src/RequestLogger.sln +++ b/src/RequestLogger.sln @@ -4,6 +4,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RequestLogger", "RequestLog EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Repository", "Repository\Repository.csproj", "{760E0EFC-C21E-46BA-9A39-C6E1D72D012E}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{39AFEA94-0387-4293-9046-6192B90B0D57}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RequestLogger.Tests", "RequestLogger.Tests\RequestLogger.Tests.csproj", "{18D1EA7C-7927-4449-B88D-A192326F11B2}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -18,5 +22,12 @@ Global {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Debug|Any CPU.Build.0 = Debug|Any CPU {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Release|Any CPU.ActiveCfg = Release|Any CPU {760E0EFC-C21E-46BA-9A39-C6E1D72D012E}.Release|Any CPU.Build.0 = Release|Any CPU + {18D1EA7C-7927-4449-B88D-A192326F11B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {18D1EA7C-7927-4449-B88D-A192326F11B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {18D1EA7C-7927-4449-B88D-A192326F11B2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {18D1EA7C-7927-4449-B88D-A192326F11B2}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {18D1EA7C-7927-4449-B88D-A192326F11B2} = {39AFEA94-0387-4293-9046-6192B90B0D57} EndGlobalSection EndGlobal diff --git a/src/RequestLogger/Controllers/WeatherForecastController.cs b/src/RequestLogger/Controllers/WeatherForecastController.cs deleted file mode 100644 index 86fa7fa..0000000 --- a/src/RequestLogger/Controllers/WeatherForecastController.cs +++ /dev/null @@ -1,32 +0,0 @@ -using Microsoft.AspNetCore.Mvc; - -namespace RequestLogger.Controllers; - -[ApiController] -[Route("[controller]")] -public class WeatherForecastController : ControllerBase -{ - private static readonly string[] Summaries = new[] - { - "Freezing", "Bracing", "Chilly", "Cool", "Mild", "Warm", "Balmy", "Hot", "Sweltering", "Scorching" - }; - - private readonly ILogger _logger; - - public WeatherForecastController(ILogger logger) - { - _logger = logger; - } - - [HttpGet(Name = "GetWeatherForecast")] - public IEnumerable Get() - { - return Enumerable.Range(1, 5).Select(index => new WeatherForecast - { - Date = DateTime.Now.AddDays(index), - TemperatureC = Random.Shared.Next(-20, 55), - Summary = Summaries[Random.Shared.Next(Summaries.Length)] - }) - .ToArray(); - } -} diff --git a/src/RequestLogger/Dto/Converters/RequestConverter.cs b/src/RequestLogger/Dto/Converters/RequestConverter.cs index 513fb52..2e428d9 100644 --- a/src/RequestLogger/Dto/Converters/RequestConverter.cs +++ b/src/RequestLogger/Dto/Converters/RequestConverter.cs @@ -1,4 +1,5 @@ -using HttpRequest = Repository.Models.HttpRequest; +using System.Text.Json; +using HttpRequest = Repository.Models.HttpRequest; namespace RequestLogger.Dto.Converters; @@ -12,9 +13,9 @@ public static HttpRequest ConvertRequest(Request request) Service = request.Service, Customer = request.Customer, Path = request.Path, - QueryParams = request.QueryParams, + QueryParams = request.QueryParams != null ? JsonSerializer.Serialize(request.QueryParams) : null, Body = request.Body, - Headers = request.Headers, + Headers = JsonSerializer.Serialize(request.Headers), RequestTime = request.RequestTime }; } diff --git a/src/RequestLogger/Dto/Request.cs b/src/RequestLogger/Dto/Request.cs index d709d36..f412eb7 100644 --- a/src/RequestLogger/Dto/Request.cs +++ b/src/RequestLogger/Dto/Request.cs @@ -1,44 +1,46 @@ -namespace RequestLogger.Dto; +using System.Collections.Specialized; + +namespace RequestLogger.Dto; public class Request { /// /// The http verb associated with the request /// - public string Verb { get; set; } = null!; + public string Verb { get; init; } = null!; /// /// The calling service /// - public string? Service { get; set; } + public string? Service { get; init; } /// /// The customer calling the service /// - public string? Customer { get; set; } + public string? Customer { get; init; } /// /// The HTTP path of the request /// - public string Path { get; set; } = null!; + public string Path { get; init; } = null!; /// /// The query parameters on the request /// - public string? QueryParams { get; set; } + public Dictionary? QueryParams { get; init; } /// /// The body of the request /// public string? Body { get; set; } - + /// /// The headers associated with a request /// - public string? Headers { get; set; } + public Dictionary Headers { get; init; } = null!; /// /// The time the request was made /// - public DateTime RequestTime { get; set; } + public DateTime RequestTime { get; init; } } \ No newline at end of file diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index d2617b5..7c58404 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -1,4 +1,6 @@ using System.Text.Json; +using System.Web; +using Microsoft.EntityFrameworkCore; using Repository; using RequestLogger.Dto; using RequestLogger.Services; @@ -24,10 +26,11 @@ builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); -builder.Services.Configure(builder.Configuration.GetSection("RequestLogger")); +builder.Services.Configure(builder.Configuration.GetSection("RequestLoggerSettings")); builder.Services.AddRequestLoggerContext(builder.Configuration); builder.Services.AddScoped(); +builder.Services.AddScoped(); var app = builder.Build(); @@ -36,9 +39,18 @@ { app.UseSwagger(); app.UseSwaggerUI(); - RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); } +using (var scope = app.Services.CreateScope()) { + + var context = scope.ServiceProvider.GetRequiredService(); + if (!context.Database.IsInMemory()) + { + RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); + } +} + + app.UseHttpsRedirection(); app.UseAuthorization(); @@ -48,36 +60,69 @@ context.Request.EnableBuffering(); context.Request.Body.Position = 0; - var rawRequestBody = await new StreamReader(context.Request.Body).ReadToEndAsync(); - //Console.WriteLine(rawRequestBody); - + var requestBody = await new StreamReader(context.Request.Body).ReadToEndAsync(); + + if (!requestBody.Equals(string.Empty)) + { + // just convert the body into minimal Json if it isn't Json + if (!IsJsonValid(requestBody)) + { + requestBody = $"{{ \"invalidJson\": \"{requestBody}\" }}"; + } + } + else + { + requestBody = null; + } + using var scoped = app.Services.CreateScope(); var loggerService = scoped.ServiceProvider.GetRequiredService(); + // gets the customer id from a path like somePath/customer//somePath + var customerId = context.Request.Path.ToString().Split('/') + .SkipWhile(p => !p.Equals("customer", StringComparison.OrdinalIgnoreCase)).Skip(1).FirstOrDefault(); + + // converts query string into a dictionary (if it has values) + var parsedQueryString = HttpUtility.ParseQueryString(context.Request.QueryString.ToString()); + var queryStringDictionary = parsedQueryString.HasKeys() ? parsedQueryString.AllKeys.ToDictionary(k => k!, k => parsedQueryString[k]!) : null; + var request = new Request() { Verb = context.Request.Method, - Service = context.Request.Scheme, - Customer = context.Request.Headers.Authorization, + Service = context.Request.Host.Value, + Customer = customerId, Path = context.Request.Path, - QueryParams = context.Request.QueryString.ToString(), - Body = rawRequestBody, - Headers = JsonSerializer.Serialize(context.Request.Headers), - RequestTime = DateTime.Now + QueryParams = queryStringDictionary, + Body = requestBody, + Headers = context.Request.Headers.ToDictionary(a => a.Key, a => a.Value.ToString()), + RequestTime = DateTime.UtcNow }; - Console.WriteLine(JsonSerializer.Serialize(request)); - - await loggerService.WriteLogMessage(request); - + var requestCompleted = await loggerService.WriteLogMessage(request); - await context.Response.WriteAsync("Ok"); + await context.Response.WriteAsync(JsonSerializer.Serialize(requestCompleted)); return; // This is never hit, but the code complains if it's not there await next(context); }); -app.MapControllers(); +bool IsJsonValid(string json) +{ + if (string.IsNullOrWhiteSpace(json)) + return false; + + try + { + using var jsonDoc = JsonDocument.Parse(json); + return true; + } + catch (JsonException) + { + return false; + } +} app.Run(); + +public partial class Program { } diff --git a/src/RequestLogger/RequestLogger.csproj b/src/RequestLogger/RequestLogger.csproj index 25c40d9..357c81e 100644 --- a/src/RequestLogger/RequestLogger.csproj +++ b/src/RequestLogger/RequestLogger.csproj @@ -10,6 +10,11 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + @@ -17,6 +22,7 @@ + diff --git a/src/RequestLogger/Services/BlacklistService.cs b/src/RequestLogger/Services/BlacklistService.cs new file mode 100644 index 0000000..6530796 --- /dev/null +++ b/src/RequestLogger/Services/BlacklistService.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using Microsoft.Extensions.Options; +using RequestLogger.Dto; +using RequestLogger.Services.Interfaces; +using RequestLogger.Settings; + +namespace RequestLogger.Services; + +public class BlacklistService : IBlacklistService +{ + private const string BodyBlacklistJson = "{ \"jsonRemoved\": \"json\" }"; + + private readonly BlacklistSettings _settings; + + + + public BlacklistService(IOptions settings) + { + _settings = settings.Value.BlacklistSettings; + } + + public (bool DoNotStore, Request? request) CheckBlacklist(Request request) + { + bool doNotStore = false; + + if (_settings.EndpointBlacklist.Any(endpoint => request.Path.StartsWith(endpoint))) + { + doNotStore = true; + // just short circuit the response as nothing else matters + return (doNotStore, null); + } + + foreach (var header in _settings.HeaderBlacklist.Where(header => request.Headers.ContainsKey(header))) + { + request.Headers.Remove(header); + } + + if (request.QueryParams != null) + { + foreach (var queryParam in _settings.QueryParamBlacklist.Where(queryParam => request.QueryParams.ContainsKey(queryParam))) + { + request.QueryParams.Remove(queryParam); + } + } + + if (_settings.BodyStorageBlacklist.Any(body => request.Path.StartsWith(body))) + { + request.Body = BodyBlacklistJson; + } + + return (doNotStore, request); + } +} \ No newline at end of file diff --git a/src/RequestLogger/Services/Interfaces/IBlacklistService.cs b/src/RequestLogger/Services/Interfaces/IBlacklistService.cs new file mode 100644 index 0000000..e824396 --- /dev/null +++ b/src/RequestLogger/Services/Interfaces/IBlacklistService.cs @@ -0,0 +1,8 @@ +using RequestLogger.Dto; + +namespace RequestLogger.Services.Interfaces; + +public interface IBlacklistService +{ + (bool DoNotStore, Request? request) CheckBlacklist(Request request); +} \ No newline at end of file diff --git a/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs b/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs index 207af4e..af16dc9 100644 --- a/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs +++ b/src/RequestLogger/Services/Interfaces/IRequestLoggerService.cs @@ -4,5 +4,5 @@ namespace RequestLogger.Services.Interfaces; public interface IRequestLoggerService { - Task WriteLogMessage(Request request); + Task WriteLogMessage(Request request); } \ No newline at end of file diff --git a/src/RequestLogger/Services/RequestLoggerService.cs b/src/RequestLogger/Services/RequestLoggerService.cs index 4cbcdc5..7e77f39 100644 --- a/src/RequestLogger/Services/RequestLoggerService.cs +++ b/src/RequestLogger/Services/RequestLoggerService.cs @@ -7,17 +7,28 @@ namespace RequestLogger.Services; public class RequestLoggerService : IRequestLoggerService { - private RequestLoggerContext _context; - public RequestLoggerService(RequestLoggerContext context) + private readonly RequestLoggerContext _context; + private readonly IBlacklistService _blacklistService; + public RequestLoggerService(RequestLoggerContext context, IBlacklistService blacklistService) { _context = context; + _blacklistService = blacklistService; } - public async Task WriteLogMessage(Request request) + public async Task WriteLogMessage(Request request) { - var httpRequest = RequestConverter.ConvertRequest(request); - await _context.Requests.AddAsync(httpRequest); + var (doNotStore, requestAfterBlacklist) = _blacklistService.CheckBlacklist(request); + + if (doNotStore) return requestAfterBlacklist; + + if (requestAfterBlacklist != null) + { + var httpRequest = RequestConverter.ConvertRequest(requestAfterBlacklist); + await _context.Requests.AddAsync(httpRequest); + } await _context.SaveChangesAsync(); + + return requestAfterBlacklist; } } \ No newline at end of file diff --git a/src/RequestLogger/Settings/BlacklistSettings.cs b/src/RequestLogger/Settings/BlacklistSettings.cs new file mode 100644 index 0000000..96e8a7f --- /dev/null +++ b/src/RequestLogger/Settings/BlacklistSettings.cs @@ -0,0 +1,12 @@ +namespace RequestLogger.Settings; + +public class BlacklistSettings +{ + public List HeaderBlacklist { get; init; } = new(); + + public List QueryParamBlacklist { get; init; } = new(); + + public List EndpointBlacklist { get; init; } = new(); + + public List BodyStorageBlacklist { get; set; } = new(); +} \ No newline at end of file diff --git a/src/RequestLogger/Settings/RequestLoggerSettings.cs b/src/RequestLogger/Settings/RequestLoggerSettings.cs index 3affcbd..7de0e21 100644 --- a/src/RequestLogger/Settings/RequestLoggerSettings.cs +++ b/src/RequestLogger/Settings/RequestLoggerSettings.cs @@ -2,5 +2,5 @@ public class RequestLoggerSettings { - + public BlacklistSettings BlacklistSettings { get; set; } = new(); } \ No newline at end of file diff --git a/src/RequestLogger/WeatherForecast.cs b/src/RequestLogger/WeatherForecast.cs deleted file mode 100644 index d347c2d..0000000 --- a/src/RequestLogger/WeatherForecast.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace RequestLogger; - -public class WeatherForecast -{ - public DateTime Date { get; set; } - - public int TemperatureC { get; set; } - - public int TemperatureF => 32 + (int)(TemperatureC / 0.5556); - - public string? Summary { get; set; } -} diff --git a/src/nginx.conf b/src/nginx.conf index 800cf1c..027daef 100644 --- a/src/nginx.conf +++ b/src/nginx.conf @@ -25,6 +25,7 @@ http { } location /mirror { + proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; internal; proxy_pass http://requestlogger$request_uri; } From 8a1dbc9a15e4ca80b552210fde27f7adde2e1462 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 6 Jul 2023 16:00:35 +0100 Subject: [PATCH 03/25] adding in github action --- .github/workflows/build-image.yml | 66 +++++++++++++++++++++++++++++++ src/nginx.conf | 1 + 2 files changed, 67 insertions(+) create mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 0000000..f7444a6 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,66 @@ +name: Build and push docker image + +on: + push: + branches: + - main + tags: + - "v*" + pull_request: + +jobs: + build-push: + runs-on: ubuntu-latest + + steps: + - id: checkout + uses: actions/checkout@v3 + + - id: setup-dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: "6.0.x" + + - id: dotnet-test + run: dotnet test ./src /p:Configuration=Release --verbosity normal + + - id: buildx + uses: docker/setup-buildx-action@v2 + + - id: docker-meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/dlcs/request-logger + context: ./src/RequestLogger + dockerfile: "Dockerfile" + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - id: docker-cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: ghcr-login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: docker_build + uses: docker/build-push-action@v4 + with: + context: . + builder: ${{ steps.buildx.outputs.name }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + push: true \ No newline at end of file diff --git a/src/nginx.conf b/src/nginx.conf index 027daef..97567d5 100644 --- a/src/nginx.conf +++ b/src/nginx.conf @@ -26,6 +26,7 @@ http { location /mirror { proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; + proxy_set_header X-service $service internal; proxy_pass http://requestlogger$request_uri; } From 8ee23584dd67f7a7ca86f3e2bc30bb0841d196a5 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 6 Jul 2023 16:05:05 +0100 Subject: [PATCH 04/25] fixing metadata --- .github/workflows/build-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index f7444a6..fa10899 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -31,8 +31,8 @@ jobs: uses: docker/metadata-action@v4 with: images: ghcr.io/dlcs/request-logger - context: ./src/RequestLogger - dockerfile: "Dockerfile" + context: ./src + dockerfile: RequestLogger/Dockerfile tags: | type=ref,event=branch type=ref,event=pr From 64a23584303701bfb003595df6c3b375794f524f Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 6 Jul 2023 16:05:40 +0100 Subject: [PATCH 05/25] actually fixing metadata --- .github/workflows/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index fa10899..4442882 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -32,7 +32,7 @@ jobs: with: images: ghcr.io/dlcs/request-logger context: ./src - dockerfile: RequestLogger/Dockerfile + dockerfile: src/RequestLogger/Dockerfile tags: | type=ref,event=branch type=ref,event=pr From b90cea6a3178c1b41a5948e061bb02c8168682ec Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Thu, 6 Jul 2023 16:13:25 +0100 Subject: [PATCH 06/25] fixing file --- .github/workflows/build-image.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 4442882..3e7b2b3 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -31,8 +31,6 @@ jobs: uses: docker/metadata-action@v4 with: images: ghcr.io/dlcs/request-logger - context: ./src - dockerfile: src/RequestLogger/Dockerfile tags: | type=ref,event=branch type=ref,event=pr @@ -59,7 +57,8 @@ jobs: - id: docker_build uses: docker/build-push-action@v4 with: - context: . + context: ./src + file: src/RequestLogger/Dockerfile builder: ${{ steps.buildx.outputs.name }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} From bd832bda81cfbfa92eb4d421c4acede16dbb2d85 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 10:39:01 +0100 Subject: [PATCH 07/25] uploading new aws settings --- .gitignore | 812 +++++++++++----------- src/RequestLogger/appsettings.DevAws.json | 15 + 2 files changed, 421 insertions(+), 406 deletions(-) create mode 100644 src/RequestLogger/appsettings.DevAws.json diff --git a/.gitignore b/.gitignore index 361fe35..f8173f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,406 +1,406 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.tlog -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files -*.ncb -*.aps - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider -*.sln.iml -*.idea/ - -# Docker -*.env - -# Project specific -*/appsettings.Development.json -*/appsettings.Docker.json \ No newline at end of file +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml +*.idea/ + +# Docker +*.env + +# Project specific +*appsettings.Development.json +*appsettings.Docker.json \ No newline at end of file diff --git a/src/RequestLogger/appsettings.DevAws.json b/src/RequestLogger/appsettings.DevAws.json new file mode 100644 index 0000000..3c1cead --- /dev/null +++ b/src/RequestLogger/appsettings.DevAws.json @@ -0,0 +1,15 @@ +{ + "RequestLoggerSettings" : { + "BlacklistSettings" : { + "HeaderBlacklist": [ + ], + "QueryParamBlacklist": [ + ], + "EndpointBlacklist": [ + "/health" + ], + "BodyStorageBlacklist": [ + ] + } + } +} From 7194af1db1a9a86a4edfad7f53662736cfe7ea7b Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 11:37:05 +0100 Subject: [PATCH 08/25] adding log message --- src/RequestLogger/Program.cs | 6 ++++ src/nginx.conf | 64 +++++++++++++++++------------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 7c58404..4138457 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -17,6 +17,8 @@ .WriteTo.Console() .CreateLogger(); +Log.Logger = logger; + // Register Serilog builder.Logging.AddSerilog(logger); @@ -41,6 +43,8 @@ app.UseSwaggerUI(); } +Log.Information("Blacklist settings: {@Settings}", app.Configuration.GetSection("RequestLoggerSettings").Get()); + using (var scope = app.Services.CreateScope()) { var context = scope.ServiceProvider.GetRequiredService(); @@ -57,6 +61,8 @@ app.Use(async (context, next) => { + var testTwo = app.Configuration.Get(); + context.Request.EnableBuffering(); context.Request.Body.Position = 0; diff --git a/src/nginx.conf b/src/nginx.conf index 97567d5..1d0c1b7 100644 --- a/src/nginx.conf +++ b/src/nginx.conf @@ -1,34 +1,32 @@ -events { -} -http { - - error_log /var/log/nginx/error.log; - access_log /var/log/nginx/access.log; - - server { - server_name ""; - # Docker resolver - resolver 127.0.0.11; - listen 80; - - location = /favicon.ico { - access_log off; - log_not_found off; - } - - location / { - mirror /mirror; - mirror_request_body on; - proxy_pass http://mockserver:1080; - #return 200 'some response!'; - # add_header Content-Type text/plain; - } - - location /mirror { - proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; - proxy_set_header X-service $service - internal; - proxy_pass http://requestlogger$request_uri; - } - } +events { +} +http { + + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log; + + server { + server_name ""; + # Docker resolver + resolver 127.0.0.11; + listen 80; + + location = /favicon.ico { + access_log off; + log_not_found off; + } + + location / { + mirror /mirror; + mirror_request_body on; + proxy_pass http://mockserver:1080; + } + + location /mirror { + proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; + #proxy_set_header X-service $service + internal; + proxy_pass http://requestlogger$request_uri; + } + } } \ No newline at end of file From 1f1a028c655c766564cbf97393090fd963ae1221 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 13:57:43 +0100 Subject: [PATCH 09/25] adding in testing appsettings --- .../Helpers/RequestLoggerAppBuilderFactory.cs | 26 ++ .../RequestLogger.Tests.csproj | 74 ++-- src/RequestLogger.Tests/Unit/ProgramTests.cs | 397 +++++++++--------- .../appsettings.Testing.json | 25 ++ src/RequestLogger/Program.cs | 20 +- src/RequestLogger/RequestLogger.csproj | 1 - 6 files changed, 293 insertions(+), 250 deletions(-) create mode 100644 src/RequestLogger.Tests/Helpers/RequestLoggerAppBuilderFactory.cs create mode 100644 src/RequestLogger.Tests/appsettings.Testing.json diff --git a/src/RequestLogger.Tests/Helpers/RequestLoggerAppBuilderFactory.cs b/src/RequestLogger.Tests/Helpers/RequestLoggerAppBuilderFactory.cs new file mode 100644 index 0000000..645f9b1 --- /dev/null +++ b/src/RequestLogger.Tests/Helpers/RequestLoggerAppBuilderFactory.cs @@ -0,0 +1,26 @@ +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; + +namespace RequestLogger.Tests.Helpers; + +public class RequestLoggerAppBuilderFactory : WebApplicationFactory + where TStartup : class +{ + //private readonly Dictionary _configuration = new(); + + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + var projectDir = Directory.GetCurrentDirectory(); + var configPath = Path.Combine(projectDir, "appsettings.Testing.json"); + + builder + .ConfigureAppConfiguration((context, conf) => + { + conf.AddJsonFile(configPath); + //conf.AddInMemoryCollection(_configuration); + }) + .UseEnvironment("Testing"); + } +} \ No newline at end of file diff --git a/src/RequestLogger.Tests/RequestLogger.Tests.csproj b/src/RequestLogger.Tests/RequestLogger.Tests.csproj index 4a24e52..dad0684 100644 --- a/src/RequestLogger.Tests/RequestLogger.Tests.csproj +++ b/src/RequestLogger.Tests/RequestLogger.Tests.csproj @@ -1,34 +1,40 @@ - - - - net6.0 - enable - enable - - false - - - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - - + + + + net6.0 + enable + enable + + false + + + + + + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + PreserveNewest + + + + diff --git a/src/RequestLogger.Tests/Unit/ProgramTests.cs b/src/RequestLogger.Tests/Unit/ProgramTests.cs index b7fbc01..2f35cc8 100644 --- a/src/RequestLogger.Tests/Unit/ProgramTests.cs +++ b/src/RequestLogger.Tests/Unit/ProgramTests.cs @@ -1,199 +1,200 @@ -using System.Text.Json; -using FakeItEasy; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.DependencyInjection; -using Repository; -using RequestLogger.Dto; -using RequestLogger.Services; -using RequestLogger.Services.Interfaces; -using AutoFixture; -using FluentAssertions; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Storage; - -#pragma warning disable CS8620 - -namespace RequestLogger.Tests.Unit; - -public class ProgramTests -{ - private readonly WebApplicationFactory _sut; - private readonly Fixture _fixture; - - public ProgramTests() - { - var root = new InMemoryDatabaseRoot(); - - _sut = new WebApplicationFactory() - .WithWebHostBuilder(builder => builder.ConfigureServices(services => - { - var descriptor = services.SingleOrDefault( - d => d.ServiceType == - typeof(DbContextOptions)); - - services.Remove(descriptor!); - services.AddScoped(); - services.AddScoped(); - services.AddDbContext(x => x.UseInMemoryDatabase("Testing", root)); - })); - - _fixture = new Fixture(); - } - - [Fact] - public void Program_CallingGet_ReturnsResponse() - { - // Arrange - var request = _fixture.Create(); - - var fakeRequestLoggerService = A.Fake(); - A.CallTo(() => fakeRequestLoggerService.WriteLogMessage(A.Ignored)).Returns(Task.FromResult(request)); - - var app = new WebApplicationFactory() - .WithWebHostBuilder(builder => builder.ConfigureServices(services => - { - services.AddScoped(_ => fakeRequestLoggerService); - })); - - var client = app.CreateClient(); - - // Act - var response = client.GetAsync("/stuff/").Result; - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - response.Content.ReadAsStringAsync().Result.Should().Contain(request.Verb); - } - - [Fact] - public void Program_MakingCall_ReturnsResponseFromDatabase() - { - // Arrange - var client = _sut.CreateClient(); - - - - // Act - var response = client.GetAsync("/stuff/").Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().BeNull(); - } - - [Fact] - public void Program_MakingCall_ReturnsCustomerSuccessfully() - { - // Arrange - var client = _sut.CreateClient(); - - // Act - var response = client.GetAsync("/stuff/customer/3425234").Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().Be("3425234"); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().BeNull(); - } - - [Fact] - public void Program_MakingCall_ReturnsQueryParamsSuccessfully() - { - // Arrange - var client = _sut.CreateClient(); - - // Act - var response = client.GetAsync("/some/url/3425234?query=something").Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().ContainKey("query"); - } - - [Fact] - public void Program_MakingPostCall_ReturnsCorrectVerb() - { - // Arrange - var client = _sut.CreateClient(); - - // Act - var response = client.PostAsync("/some/uri", new StringContent("")).Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); - } - - [Fact] - public void Program_MakingPostCallWithJsonBody_ReturnsCorrectBody() - { - // Arrange - var client = _sut.CreateClient(); - var jsonBody = "{\"test\": \"test\"}"; - - // Act - var response = client.PostAsync("/some/uri", new StringContent(jsonBody)).Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().Be(jsonBody); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); - } - - [Fact] - public void Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() - { - // Arrange - var client = _sut.CreateClient(); - var requestBody = "test"; - var jsonBody = "{ \"invalidJson\": \"test\" }"; - - // Act - var response = client.PostAsync("/some/uri", new StringContent(requestBody)).Result; - - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); - - //Assert - response.Should().NotBeNull(); - response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().Be(jsonBody); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); - } +using System.Text.Json; +using FakeItEasy; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.DependencyInjection; +using Repository; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Services.Interfaces; +using AutoFixture; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using RequestLogger.Tests.Helpers; + +#pragma warning disable CS8620 + +namespace RequestLogger.Tests.Unit; + +public class ProgramTests +{ + private readonly WebApplicationFactory _sut; + private readonly Fixture _fixture; + + public ProgramTests() + { + var root = new InMemoryDatabaseRoot(); + + _sut = new RequestLoggerAppBuilderFactory() + .WithWebHostBuilder(builder => builder.ConfigureServices(services => + { + var descriptor = services.SingleOrDefault( + d => d.ServiceType == + typeof(DbContextOptions)); + + services.Remove(descriptor!); + services.AddScoped(); + services.AddScoped(); + services.AddDbContext(x => x.UseInMemoryDatabase("Testing", root)); + })); + + _fixture = new Fixture(); + } + + [Fact] + public void Program_CallingGet_ReturnsResponse() + { + // Arrange + var request = _fixture.Create(); + + var fakeRequestLoggerService = A.Fake(); + A.CallTo(() => fakeRequestLoggerService.WriteLogMessage(A.Ignored)).Returns(Task.FromResult(request)); + + var app = new WebApplicationFactory() + .WithWebHostBuilder(builder => builder.ConfigureServices(services => + { + services.AddScoped(_ => fakeRequestLoggerService); + })); + + var client = app.CreateClient(); + + // Act + var response = client.GetAsync("/stuff/").Result; + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + response.Content.ReadAsStringAsync().Result.Should().Contain(request.Verb); + } + + [Fact] + public void Program_MakingCall_ReturnsResponseFromDatabase() + { + // Arrange + var client = _sut.CreateClient(); + + + + // Act + var response = client.GetAsync("/stuff/").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingCall_ReturnsCustomerSuccessfully() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.GetAsync("/stuff/customer/3425234").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().Be("3425234"); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingCall_ReturnsQueryParamsSuccessfully() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.GetAsync("/some/url/3425234?query=something").Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("GET"); + request?.QueryParams.Should().ContainKey("query"); + } + + [Fact] + public void Program_MakingPostCall_ReturnsCorrectVerb() + { + // Arrange + var client = _sut.CreateClient(); + + // Act + var response = client.PostAsync("/some/uri", new StringContent("")).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().BeNull(); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingPostCallWithJsonBody_ReturnsCorrectBody() + { + // Arrange + var client = _sut.CreateClient(); + var jsonBody = "{\"test\": \"test\"}"; + + // Act + var response = client.PostAsync("/some/uri", new StringContent(jsonBody)).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().Be(jsonBody); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } + + [Fact] + public void Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() + { + // Arrange + var client = _sut.CreateClient(); + var requestBody = "test"; + var jsonBody = "{ \"invalidJson\": \"test\" }"; + + // Act + var response = client.PostAsync("/some/uri", new StringContent(requestBody)).Result; + + var responseContent = response.Content.ReadAsStringAsync().Result; + var request = JsonSerializer.Deserialize(responseContent); + + //Assert + response.Should().NotBeNull(); + response.IsSuccessStatusCode.Should().BeTrue(); + request?.Customer.Should().BeNull(); + request?.Body.Should().Be(jsonBody); + request?.Verb.Should().Be("POST"); + request?.QueryParams.Should().BeNull(); + } } \ No newline at end of file diff --git a/src/RequestLogger.Tests/appsettings.Testing.json b/src/RequestLogger.Tests/appsettings.Testing.json new file mode 100644 index 0000000..f6e8c35 --- /dev/null +++ b/src/RequestLogger.Tests/appsettings.Testing.json @@ -0,0 +1,25 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "RequestLoggerSettings" : { + "BlacklistSettings" : { + "HeaderBlacklist": [ + "Blacklisted" + ], + "QueryParamBlacklist": [ + "blacklist" + ], + "EndpointBlacklist": [ + "/blacklist/test" + ], + "BodyStorageBlacklist": [ + "/blacklist/noBody" + ] + } + }, + "RunMigrations": false +} diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 4138457..e0f1040 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -43,26 +43,16 @@ app.UseSwaggerUI(); } -Log.Information("Blacklist settings: {@Settings}", app.Configuration.GetSection("RequestLoggerSettings").Get()); - -using (var scope = app.Services.CreateScope()) { - - var context = scope.ServiceProvider.GetRequiredService(); - if (!context.Database.IsInMemory()) - { - RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); - } -} +Log.Information("Request logger settings: {@Settings}", app.Configuration.GetSection("RequestLoggerSettings").Get()); +RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); app.UseHttpsRedirection(); app.UseAuthorization(); -app.Use(async (context, next) => +app.Run(async (context) => { - var testTwo = app.Configuration.Get(); - context.Request.EnableBuffering(); context.Request.Body.Position = 0; @@ -107,10 +97,6 @@ var requestCompleted = await loggerService.WriteLogMessage(request); await context.Response.WriteAsync(JsonSerializer.Serialize(requestCompleted)); - return; - - // This is never hit, but the code complains if it's not there - await next(context); }); bool IsJsonValid(string json) diff --git a/src/RequestLogger/RequestLogger.csproj b/src/RequestLogger/RequestLogger.csproj index 357c81e..8182311 100644 --- a/src/RequestLogger/RequestLogger.csproj +++ b/src/RequestLogger/RequestLogger.csproj @@ -14,7 +14,6 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - From 7214139390cec76ba4a9fb463cafac47708222dd Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 14:25:26 +0100 Subject: [PATCH 10/25] adding nginx to build --- .github/workflows/build-image.yml | 55 +++++++++++ .gitignore | 4 +- compose/docker-compose-local.yml | 124 ++++++++++++------------ nginx/nginx.dlcs.Dockerfile | 3 + src/nginx.conf => nginx/nginx.dlcs.conf | 0 nginx/nginx.local.Dockerfile | 3 + nginx/nginx.local.conf | 32 ++++++ 7 files changed, 159 insertions(+), 62 deletions(-) create mode 100644 nginx/nginx.dlcs.Dockerfile rename src/nginx.conf => nginx/nginx.dlcs.conf (100%) create mode 100644 nginx/nginx.local.Dockerfile create mode 100644 nginx/nginx.local.conf diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 3e7b2b3..157c4d9 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -62,4 +62,59 @@ jobs: builder: ${{ steps.buildx.outputs.name }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} + push: true + + build-push-nginx: + runs-on: ubuntu-latest + + steps: + - id: checkout + uses: actions/checkout@v3 + + - id: setup-dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: "6.0.x" + + - id: dotnet-test + run: dotnet test ./src /p:Configuration=Release --verbosity normal + + - id: buildx + uses: docker/setup-buildx-action@v2 + + - id: docker-meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/dlcs/request-logger-nginx + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - id: docker-cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: ghcr-login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: docker_build + uses: docker/build-push-action@v4 + with: + context: ./nginx + file: nginx.dlcs.Dockerfile + builder: ${{ steps.buildx.outputs.name }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} push: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index f8173f4..3a1901a 100644 --- a/.gitignore +++ b/.gitignore @@ -402,5 +402,5 @@ FodyWeavers.xsd *.env # Project specific -*appsettings.Development.json -*appsettings.Docker.json \ No newline at end of file +*.Development.json +*.Docker.json \ No newline at end of file diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 6a9ae46..672f50d 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -1,61 +1,65 @@ -version: '3' -name: request-logger - -services: - web: - image: nginx - container_name: nginx - volumes: - - ../src/nginx.conf:/etc/nginx/nginx.conf:ro - command: [nginx-debug, '-g', 'daemon off;'] - ports: - - "8080:80" - environment: - - NGINX_HOST=foobar.com - - NGINX_PORT=80 - networks: - - nginx.docker - - requestlogger: - container_name: request-logger - build: - context: ../src - dockerfile: RequestLogger\Dockerfile - networks: - - nginx.docker - env_file: - - .env - - mockserver: - image: mockserver/mockserver:latest - container_name: mock-server - environment: - MOCKSERVER_WATCH_INITIALIZATION_JSON: "true" - MOCKSERVER_PROPERTY_FILE: /config/mockserver.properties - MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializerJson.json - volumes: - - ../src/MockServerConfig/initializerJson.json:/config/initializerJson.json:ro - networks: - - nginx.docker - - postgres: - image: postgres:12.15 - container_name: postgres - hostname: postgres - ports: - - "5452:5432" - volumes: - - rl_postgres_data:/var/lib/postgresql/data - - rl_postgres_data_backups:/backups - env_file: - - .env - networks: - - nginx.docker - -networks: - nginx.docker: - name: nginx.docker - -volumes: - rl_postgres_data: {} +version: '3' +name: request-logger + +services: + web: + container_name: nginx + build: + context: ../nginx + dockerfile: nginx.local.Dockerfile + # image: nginx + # container_name: nginx + # volumes: + # - ../src/nginx.conf:/etc/nginx/nginx.conf:ro + command: [nginx-debug, '-g', 'daemon off;'] + ports: + - "8080:80" + environment: + - NGINX_HOST=foobar.com + - NGINX_PORT=80 + networks: + - nginx.docker + + requestlogger: + container_name: request-logger + build: + context: ../src + dockerfile: RequestLogger\Dockerfile + networks: + - nginx.docker + env_file: + - .env + + mockserver: + image: mockserver/mockserver:latest + container_name: mock-server + environment: + MOCKSERVER_WATCH_INITIALIZATION_JSON: "true" + MOCKSERVER_PROPERTY_FILE: /config/mockserver.properties + MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializerJson.json + volumes: + - ../src/MockServerConfig/initializerJson.json:/config/initializerJson.json:ro + networks: + - nginx.docker + + postgres: + image: postgres:12.15 + container_name: postgres + hostname: postgres + ports: + - "5452:5432" + volumes: + - rl_postgres_data:/var/lib/postgresql/data + - rl_postgres_data_backups:/backups + env_file: + - .env + networks: + - nginx.docker + +networks: + nginx.docker: + name: nginx.docker + +volumes: + rl_postgres_data: {} rl_postgres_data_backups: {} \ No newline at end of file diff --git a/nginx/nginx.dlcs.Dockerfile b/nginx/nginx.dlcs.Dockerfile new file mode 100644 index 0000000..8c10c10 --- /dev/null +++ b/nginx/nginx.dlcs.Dockerfile @@ -0,0 +1,3 @@ +FROM nginx +RUN rm /etc/nginx/nginx.conf /etc/nginx/conf.d/default.conf +COPY ./nginx.dlcs.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/src/nginx.conf b/nginx/nginx.dlcs.conf similarity index 100% rename from src/nginx.conf rename to nginx/nginx.dlcs.conf diff --git a/nginx/nginx.local.Dockerfile b/nginx/nginx.local.Dockerfile new file mode 100644 index 0000000..7269f2c --- /dev/null +++ b/nginx/nginx.local.Dockerfile @@ -0,0 +1,3 @@ +FROM nginx +RUN rm /etc/nginx/nginx.conf /etc/nginx/conf.d/default.conf +COPY ./nginx.local.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/nginx/nginx.local.conf b/nginx/nginx.local.conf new file mode 100644 index 0000000..1d0c1b7 --- /dev/null +++ b/nginx/nginx.local.conf @@ -0,0 +1,32 @@ +events { +} +http { + + error_log /var/log/nginx/error.log; + access_log /var/log/nginx/access.log; + + server { + server_name ""; + # Docker resolver + resolver 127.0.0.11; + listen 80; + + location = /favicon.ico { + access_log off; + log_not_found off; + } + + location / { + mirror /mirror; + mirror_request_body on; + proxy_pass http://mockserver:1080; + } + + location /mirror { + proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; + #proxy_set_header X-service $service + internal; + proxy_pass http://requestlogger$request_uri; + } + } +} \ No newline at end of file From 20ba16266d6eec7a34b73c935432da7cfb9a19dc Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 14:29:11 +0100 Subject: [PATCH 11/25] fixing build --- .github/workflows/build-image.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 157c4d9..d3a915b 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -71,14 +71,6 @@ jobs: - id: checkout uses: actions/checkout@v3 - - id: setup-dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: "6.0.x" - - - id: dotnet-test - run: dotnet test ./src /p:Configuration=Release --verbosity normal - - id: buildx uses: docker/setup-buildx-action@v2 @@ -113,7 +105,7 @@ jobs: uses: docker/build-push-action@v4 with: context: ./nginx - file: nginx.dlcs.Dockerfile + file: nginx/nginx.dlcs.Dockerfile builder: ${{ steps.buildx.outputs.name }} tags: ${{ steps.docker-meta.outputs.tags }} labels: ${{ steps.docker-meta.outputs.labels }} From 9fe2fad067af817cc44eb28095111c76ad97f66b Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 16:32:09 +0100 Subject: [PATCH 12/25] updating to use new nginx config --- .github/workflows/build-image.yml | 112 ------------------------------ compose/docker-compose-local.yml | 9 ++- nginx/docker-entrypoint.sh | 6 ++ nginx/nginx.Dockerfile | 7 ++ nginx/nginx.default.conf | 28 ++++++++ nginx/nginx.dlcs.Dockerfile | 3 - nginx/nginx.dlcs.conf | 32 --------- nginx/nginx.local.Dockerfile | 3 - nginx/nginx.local.conf | 32 --------- 9 files changed, 45 insertions(+), 187 deletions(-) delete mode 100644 .github/workflows/build-image.yml create mode 100644 nginx/docker-entrypoint.sh create mode 100644 nginx/nginx.Dockerfile create mode 100644 nginx/nginx.default.conf delete mode 100644 nginx/nginx.dlcs.Dockerfile delete mode 100644 nginx/nginx.dlcs.conf delete mode 100644 nginx/nginx.local.Dockerfile delete mode 100644 nginx/nginx.local.conf diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index d3a915b..0000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Build and push docker image - -on: - push: - branches: - - main - tags: - - "v*" - pull_request: - -jobs: - build-push: - runs-on: ubuntu-latest - - steps: - - id: checkout - uses: actions/checkout@v3 - - - id: setup-dotnet - uses: actions/setup-dotnet@v3 - with: - dotnet-version: "6.0.x" - - - id: dotnet-test - run: dotnet test ./src /p:Configuration=Release --verbosity normal - - - id: buildx - uses: docker/setup-buildx-action@v2 - - - id: docker-meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/dlcs/request-logger - tags: | - type=ref,event=branch - type=ref,event=pr - type=sha,format=long - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - - id: docker-cache - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - id: ghcr-login - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - id: docker_build - uses: docker/build-push-action@v4 - with: - context: ./src - file: src/RequestLogger/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: ${{ steps.docker-meta.outputs.tags }} - labels: ${{ steps.docker-meta.outputs.labels }} - push: true - - build-push-nginx: - runs-on: ubuntu-latest - - steps: - - id: checkout - uses: actions/checkout@v3 - - - id: buildx - uses: docker/setup-buildx-action@v2 - - - id: docker-meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/dlcs/request-logger-nginx - tags: | - type=ref,event=branch - type=ref,event=pr - type=sha,format=long - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - - id: docker-cache - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - - id: ghcr-login - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - id: docker_build - uses: docker/build-push-action@v4 - with: - context: ./nginx - file: nginx/nginx.dlcs.Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: ${{ steps.docker-meta.outputs.tags }} - labels: ${{ steps.docker-meta.outputs.labels }} - push: true \ No newline at end of file diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 672f50d..9e9ffa9 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -6,17 +6,16 @@ services: container_name: nginx build: context: ../nginx - dockerfile: nginx.local.Dockerfile - # image: nginx - # container_name: nginx - # volumes: - # - ../src/nginx.conf:/etc/nginx/nginx.conf:ro + dockerfile: nginx.Dockerfile command: [nginx-debug, '-g', 'daemon off;'] ports: - "8080:80" environment: - NGINX_HOST=foobar.com - NGINX_PORT=80 + - EXTERNAL_PORT=80 + - NGINX_PROXY_PASS_MIRROR_LOCATION=http://requestlogger + - NGINX_PROXY_PASS_LOCATION=http://mockserver:1080 networks: - nginx.docker diff --git a/nginx/docker-entrypoint.sh b/nginx/docker-entrypoint.sh new file mode 100644 index 0000000..a663bbd --- /dev/null +++ b/nginx/docker-entrypoint.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +set -eu + +envsubst '${EXTERNAL_PORT} ${NGINX_PROXY_PASS_LOCATION} ${NGINX_PROXY_PASS_MIRROR_LOCATION}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf + +exec "$@" \ No newline at end of file diff --git a/nginx/nginx.Dockerfile b/nginx/nginx.Dockerfile new file mode 100644 index 0000000..4119c9a --- /dev/null +++ b/nginx/nginx.Dockerfile @@ -0,0 +1,7 @@ +FROM nginx + +COPY ./nginx.default.conf /etc/nginx/conf.d/default.conf.template + +COPY docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/nginx/nginx.default.conf b/nginx/nginx.default.conf new file mode 100644 index 0000000..143edce --- /dev/null +++ b/nginx/nginx.default.conf @@ -0,0 +1,28 @@ + +error_log /var/log/nginx/error.log; +access_log /var/log/nginx/access.log; + +server { + server_name ""; + # Docker resolver + resolver 127.0.0.11; + listen ${EXTERNAL_PORT}; + + location = /favicon.ico { + access_log off; + log_not_found off; + } + + location / { + mirror /mirror; + mirror_request_body on; + proxy_pass ${NGINX_PROXY_PASS_LOCATION}; + } + + location /mirror { + proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; + #proxy_set_header X-service $service + internal; + proxy_pass ${NGINX_PROXY_PASS_MIRROR_LOCATION}$request_uri; + } +} \ No newline at end of file diff --git a/nginx/nginx.dlcs.Dockerfile b/nginx/nginx.dlcs.Dockerfile deleted file mode 100644 index 8c10c10..0000000 --- a/nginx/nginx.dlcs.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM nginx -RUN rm /etc/nginx/nginx.conf /etc/nginx/conf.d/default.conf -COPY ./nginx.dlcs.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/nginx/nginx.dlcs.conf b/nginx/nginx.dlcs.conf deleted file mode 100644 index 1d0c1b7..0000000 --- a/nginx/nginx.dlcs.conf +++ /dev/null @@ -1,32 +0,0 @@ -events { -} -http { - - error_log /var/log/nginx/error.log; - access_log /var/log/nginx/access.log; - - server { - server_name ""; - # Docker resolver - resolver 127.0.0.11; - listen 80; - - location = /favicon.ico { - access_log off; - log_not_found off; - } - - location / { - mirror /mirror; - mirror_request_body on; - proxy_pass http://mockserver:1080; - } - - location /mirror { - proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; - #proxy_set_header X-service $service - internal; - proxy_pass http://requestlogger$request_uri; - } - } -} \ No newline at end of file diff --git a/nginx/nginx.local.Dockerfile b/nginx/nginx.local.Dockerfile deleted file mode 100644 index 7269f2c..0000000 --- a/nginx/nginx.local.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM nginx -RUN rm /etc/nginx/nginx.conf /etc/nginx/conf.d/default.conf -COPY ./nginx.local.conf /etc/nginx/nginx.conf \ No newline at end of file diff --git a/nginx/nginx.local.conf b/nginx/nginx.local.conf deleted file mode 100644 index 1d0c1b7..0000000 --- a/nginx/nginx.local.conf +++ /dev/null @@ -1,32 +0,0 @@ -events { -} -http { - - error_log /var/log/nginx/error.log; - access_log /var/log/nginx/access.log; - - server { - server_name ""; - # Docker resolver - resolver 127.0.0.11; - listen 80; - - location = /favicon.ico { - access_log off; - log_not_found off; - } - - location / { - mirror /mirror; - mirror_request_body on; - proxy_pass http://mockserver:1080; - } - - location /mirror { - proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; - #proxy_set_header X-service $service - internal; - proxy_pass http://requestlogger$request_uri; - } - } -} \ No newline at end of file From f0bc24872133d22f968c094469f42f6ecec2a49d Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 16:52:17 +0100 Subject: [PATCH 13/25] adding in ability to set service based on a header value --- .github/workflows/build-images.yml | 112 +++++++++++++++++++++++++++++ compose/docker-compose-local.yml | 1 + nginx/docker-entrypoint.sh | 2 +- nginx/nginx.default.conf | 2 +- src/RequestLogger/Program.cs | 4 +- 5 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-images.yml diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml new file mode 100644 index 0000000..c209083 --- /dev/null +++ b/.github/workflows/build-images.yml @@ -0,0 +1,112 @@ +name: Build and push docker images + +on: + push: + branches: + - main + tags: + - "v*" + pull_request: + +jobs: + build-push: + runs-on: ubuntu-latest + + steps: + - id: checkout + uses: actions/checkout@v3 + + - id: setup-dotnet + uses: actions/setup-dotnet@v3 + with: + dotnet-version: "6.0.x" + + - id: dotnet-test + run: dotnet test ./src /p:Configuration=Release --verbosity normal + + - id: buildx + uses: docker/setup-buildx-action@v2 + + - id: docker-meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/dlcs/request-logger + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - id: docker-cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: ghcr-login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: docker_build + uses: docker/build-push-action@v4 + with: + context: ./src + file: src/RequestLogger/Dockerfile + builder: ${{ steps.buildx.outputs.name }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + push: true + + build-push-nginx: + runs-on: ubuntu-latest + + steps: + - id: checkout + uses: actions/checkout@v3 + + - id: buildx + uses: docker/setup-buildx-action@v2 + + - id: docker-meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/dlcs/request-logger-nginx + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - id: docker-cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - id: ghcr-login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - id: docker_build + uses: docker/build-push-action@v4 + with: + context: ./nginx + file: nginx/nginx.Dockerfile + builder: ${{ steps.buildx.outputs.name }} + tags: ${{ steps.docker-meta.outputs.tags }} + labels: ${{ steps.docker-meta.outputs.labels }} + push: true \ No newline at end of file diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 9e9ffa9..525b0f0 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -16,6 +16,7 @@ services: - EXTERNAL_PORT=80 - NGINX_PROXY_PASS_MIRROR_LOCATION=http://requestlogger - NGINX_PROXY_PASS_LOCATION=http://mockserver:1080 + - SERVICE=DLCS-local networks: - nginx.docker diff --git a/nginx/docker-entrypoint.sh b/nginx/docker-entrypoint.sh index a663bbd..6bb496f 100644 --- a/nginx/docker-entrypoint.sh +++ b/nginx/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh set -eu -envsubst '${EXTERNAL_PORT} ${NGINX_PROXY_PASS_LOCATION} ${NGINX_PROXY_PASS_MIRROR_LOCATION}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +envsubst '${EXTERNAL_PORT} ${NGINX_PROXY_PASS_LOCATION} ${NGINX_PROXY_PASS_MIRROR_LOCATION} ${SERVICE}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" \ No newline at end of file diff --git a/nginx/nginx.default.conf b/nginx/nginx.default.conf index 143edce..46ee290 100644 --- a/nginx/nginx.default.conf +++ b/nginx/nginx.default.conf @@ -21,7 +21,7 @@ server { location /mirror { proxy_set_header X-Forwarded-For "$http_x_forwarded_for, $realip_remote_addr"; - #proxy_set_header X-service $service + proxy_set_header X-Service ${SERVICE}; internal; proxy_pass ${NGINX_PROXY_PASS_MIRROR_LOCATION}$request_uri; } diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index e0f1040..15e4824 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -82,10 +82,12 @@ var parsedQueryString = HttpUtility.ParseQueryString(context.Request.QueryString.ToString()); var queryStringDictionary = parsedQueryString.HasKeys() ? parsedQueryString.AllKeys.ToDictionary(k => k!, k => parsedQueryString[k]!) : null; + var service = context.Request.Headers.TryGetValue("X-Service", out var header) ? header.ToString() : context.Request.Host.Value; + var request = new Request() { Verb = context.Request.Method, - Service = context.Request.Host.Value, + Service = service, Customer = customerId, Path = context.Request.Path, QueryParams = queryStringDictionary, From 67e011414da75c5d6a67107f5016c1e05565089f Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Fri, 7 Jul 2023 17:00:58 +0100 Subject: [PATCH 14/25] adding in request blacklists --- .../Settings/BlacklistSettings.cs | 12 +++++++++ .../Settings/RequestLoggerSettings.cs | 13 +++++---- src/RequestLogger/appsettings.Docker.json | 27 +++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/RequestLogger/appsettings.Docker.json diff --git a/src/RequestLogger/Settings/BlacklistSettings.cs b/src/RequestLogger/Settings/BlacklistSettings.cs index 96e8a7f..7ba32e5 100644 --- a/src/RequestLogger/Settings/BlacklistSettings.cs +++ b/src/RequestLogger/Settings/BlacklistSettings.cs @@ -2,11 +2,23 @@ public class BlacklistSettings { + /// + /// Header blacklist + /// public List HeaderBlacklist { get; init; } = new(); + /// + /// Query parameter blacklist + /// public List QueryParamBlacklist { get; init; } = new(); + /// + /// Endpoint blacklist + /// public List EndpointBlacklist { get; init; } = new(); + /// + /// Request body blacklist + /// public List BodyStorageBlacklist { get; set; } = new(); } \ No newline at end of file diff --git a/src/RequestLogger/Settings/RequestLoggerSettings.cs b/src/RequestLogger/Settings/RequestLoggerSettings.cs index 7de0e21..5d9eba4 100644 --- a/src/RequestLogger/Settings/RequestLoggerSettings.cs +++ b/src/RequestLogger/Settings/RequestLoggerSettings.cs @@ -1,6 +1,9 @@ -namespace RequestLogger.Settings; - -public class RequestLoggerSettings -{ - public BlacklistSettings BlacklistSettings { get; set; } = new(); +namespace RequestLogger.Settings; + +public class RequestLoggerSettings +{ + /// + /// Settings related to configuring blacklists + /// + public BlacklistSettings BlacklistSettings { get; set; } = new(); } \ No newline at end of file diff --git a/src/RequestLogger/appsettings.Docker.json b/src/RequestLogger/appsettings.Docker.json new file mode 100644 index 0000000..0e5883a --- /dev/null +++ b/src/RequestLogger/appsettings.Docker.json @@ -0,0 +1,27 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + } + }, + "RequestLoggerSettings" : { + "BlacklistSettings" : { + "HeaderBlacklist": [ + "Blacklisted" + ], + "QueryParamBlacklist": [ + "blacklist" + ], + "EndpointBlacklist": [ + "/blacklist/test" + ], + "BodyStorageBlacklist": [ + "/blacklist/noBody" + ] + } + }, + "ConnectionStrings": { + }, + "RunMigrations": true +} From f0db9556a1127d7a0d077e80b915def44f854656 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 09:48:30 +0100 Subject: [PATCH 15/25] updating dockerfile permissions --- nginx/nginx.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nginx/nginx.Dockerfile b/nginx/nginx.Dockerfile index 4119c9a..51e47c3 100644 --- a/nginx/nginx.Dockerfile +++ b/nginx/nginx.Dockerfile @@ -3,5 +3,6 @@ FROM nginx COPY ./nginx.default.conf /etc/nginx/conf.d/default.conf.template COPY docker-entrypoint.sh / +RUN chmod +x entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file From 7b7fe047e1ebd2e6757db0233e73c8199913fe61 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 09:50:46 +0100 Subject: [PATCH 16/25] renaming file to correct --- nginx/nginx.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.Dockerfile b/nginx/nginx.Dockerfile index 51e47c3..f3bdd15 100644 --- a/nginx/nginx.Dockerfile +++ b/nginx/nginx.Dockerfile @@ -3,6 +3,6 @@ FROM nginx COPY ./nginx.default.conf /etc/nginx/conf.d/default.conf.template COPY docker-entrypoint.sh / -RUN chmod +x entrypoint.sh +RUN chmod +x docker-entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file From 2cb9ec8c7db3c7ee19f243e0ee8eb6ab9d3d710a Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 10:13:43 +0100 Subject: [PATCH 17/25] removing no longer required settings --- src/RequestLogger/appsettings.DevAws.json | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 src/RequestLogger/appsettings.DevAws.json diff --git a/src/RequestLogger/appsettings.DevAws.json b/src/RequestLogger/appsettings.DevAws.json deleted file mode 100644 index 3c1cead..0000000 --- a/src/RequestLogger/appsettings.DevAws.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "RequestLoggerSettings" : { - "BlacklistSettings" : { - "HeaderBlacklist": [ - ], - "QueryParamBlacklist": [ - ], - "EndpointBlacklist": [ - "/health" - ], - "BodyStorageBlacklist": [ - ] - } - } -} From 418a5ab98ea8dcf8d716f9bd7934888194ffefd1 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 14:37:24 +0100 Subject: [PATCH 18/25] seeing if changing the expose works --- nginx/nginx.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/nginx/nginx.Dockerfile b/nginx/nginx.Dockerfile index f3bdd15..9ea8677 100644 --- a/nginx/nginx.Dockerfile +++ b/nginx/nginx.Dockerfile @@ -1,4 +1,5 @@ FROM nginx +EXPOSE 80 COPY ./nginx.default.conf /etc/nginx/conf.d/default.conf.template From cae391bff7b2a11cd78f5a0f9195a7b543a883db Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 14:45:25 +0100 Subject: [PATCH 19/25] setting compose to use 8080 by default --- compose/docker-compose-local.yml | 4 ++-- nginx/nginx.Dockerfile | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 525b0f0..91604eb 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -9,11 +9,11 @@ services: dockerfile: nginx.Dockerfile command: [nginx-debug, '-g', 'daemon off;'] ports: - - "8080:80" + - "8080:8080" environment: - NGINX_HOST=foobar.com - NGINX_PORT=80 - - EXTERNAL_PORT=80 + - EXTERNAL_PORT=8080 - NGINX_PROXY_PASS_MIRROR_LOCATION=http://requestlogger - NGINX_PROXY_PASS_LOCATION=http://mockserver:1080 - SERVICE=DLCS-local diff --git a/nginx/nginx.Dockerfile b/nginx/nginx.Dockerfile index 9ea8677..f3bdd15 100644 --- a/nginx/nginx.Dockerfile +++ b/nginx/nginx.Dockerfile @@ -1,5 +1,4 @@ FROM nginx -EXPOSE 80 COPY ./nginx.default.conf /etc/nginx/conf.d/default.conf.template From fcf6cb01d9af93bd7781e32c5b26398c5823ab01 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Mon, 10 Jul 2023 15:51:16 +0100 Subject: [PATCH 20/25] allowing the resolver to be set from environment variables --- compose/docker-compose-local.yml | 1 + nginx/docker-entrypoint.sh | 2 +- nginx/nginx.default.conf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/docker-compose-local.yml b/compose/docker-compose-local.yml index 91604eb..d5b1836 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose-local.yml @@ -17,6 +17,7 @@ services: - NGINX_PROXY_PASS_MIRROR_LOCATION=http://requestlogger - NGINX_PROXY_PASS_LOCATION=http://mockserver:1080 - SERVICE=DLCS-local + - NGINX_RESOLVER=127.0.0.11 networks: - nginx.docker diff --git a/nginx/docker-entrypoint.sh b/nginx/docker-entrypoint.sh index 6bb496f..4e2ddd1 100644 --- a/nginx/docker-entrypoint.sh +++ b/nginx/docker-entrypoint.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh set -eu -envsubst '${EXTERNAL_PORT} ${NGINX_PROXY_PASS_LOCATION} ${NGINX_PROXY_PASS_MIRROR_LOCATION} ${SERVICE}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf +envsubst '${EXTERNAL_PORT} ${NGINX_PROXY_PASS_LOCATION} ${NGINX_PROXY_PASS_MIRROR_LOCATION} ${SERVICE} ${NGINX_RESOLVER}' < /etc/nginx/conf.d/default.conf.template > /etc/nginx/conf.d/default.conf exec "$@" \ No newline at end of file diff --git a/nginx/nginx.default.conf b/nginx/nginx.default.conf index 46ee290..9ca01bc 100644 --- a/nginx/nginx.default.conf +++ b/nginx/nginx.default.conf @@ -5,7 +5,7 @@ access_log /var/log/nginx/access.log; server { server_name ""; # Docker resolver - resolver 127.0.0.11; + resolver ${NGINX_RESOLVER}; listen ${EXTERNAL_PORT}; location = /favicon.ico { From 6d33ee57da8e68f3dc98cf8f6e1e1455e96db912 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 11 Jul 2023 13:45:28 +0100 Subject: [PATCH 21/25] pushing up code review changes --- ...r-compose-local.yml => docker-compose.yml} | 14 +- src/Repository/RequestLoggerContext.cs | 77 +++++------ src/RequestLogger.Tests/Unit/ProgramTests.cs | 100 +++++++------- .../Unit/RequestLoggerServiceTests.cs | 126 +++++++++--------- .../Dto/Converters/RequestConverter.cs | 42 +++--- src/RequestLogger/Program.cs | 106 +++++++++++---- src/RequestLogger/RequestLogger.csproj | 3 + 7 files changed, 253 insertions(+), 215 deletions(-) rename compose/{docker-compose-local.yml => docker-compose.yml} (84%) diff --git a/compose/docker-compose-local.yml b/compose/docker-compose.yml similarity index 84% rename from compose/docker-compose-local.yml rename to compose/docker-compose.yml index d5b1836..3dd80da 100644 --- a/compose/docker-compose-local.yml +++ b/compose/docker-compose.yml @@ -18,16 +18,14 @@ services: - NGINX_PROXY_PASS_LOCATION=http://mockserver:1080 - SERVICE=DLCS-local - NGINX_RESOLVER=127.0.0.11 - networks: - - nginx.docker requestlogger: container_name: request-logger build: context: ../src dockerfile: RequestLogger\Dockerfile - networks: - - nginx.docker + environment: + - ConnectionStrings__PostgreSQLConnection=Server=postgres;Port=5432;Database=${POSTGRES_DB};User Id=${POSTGRES_USER};Password=${POSTGRES_PASSWORD}; env_file: - .env @@ -40,8 +38,6 @@ services: MOCKSERVER_INITIALIZATION_JSON_PATH: /config/initializerJson.json volumes: - ../src/MockServerConfig/initializerJson.json:/config/initializerJson.json:ro - networks: - - nginx.docker postgres: image: postgres:12.15 @@ -54,12 +50,6 @@ services: - rl_postgres_data_backups:/backups env_file: - .env - networks: - - nginx.docker - -networks: - nginx.docker: - name: nginx.docker volumes: rl_postgres_data: {} diff --git a/src/Repository/RequestLoggerContext.cs b/src/Repository/RequestLoggerContext.cs index cc9fa93..76723a5 100644 --- a/src/Repository/RequestLoggerContext.cs +++ b/src/Repository/RequestLoggerContext.cs @@ -1,43 +1,36 @@ -using Microsoft.EntityFrameworkCore; -using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; -using Repository.Models; - - -namespace Repository; - -public class RequestLoggerContext : DbContext -{ - /// - /// Context class for entity framework - /// - public RequestLoggerContext() - { - } - - /// - /// Context class for entity framework - /// - /// The db context options - public RequestLoggerContext(DbContextOptions options) - : base(options) - { - } - - protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) - { - } - - protected override void OnModelCreating(ModelBuilder modelBuilder) - { - modelBuilder.Entity(builder => - { - builder.HasKey(i => i.Id); - builder.HasAnnotation("Npgsql:ValueGenerationStrategy", - NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); - builder.Property(p => p.Body).HasColumnType("jsonb"); - builder.Property(p => p.Headers).HasColumnType("jsonb"); - }); - } - - public virtual DbSet Requests { get; set; } = null!; +using Microsoft.EntityFrameworkCore; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Repository.Models; + + +namespace Repository; + +public class RequestLoggerContext : DbContext +{ + /// + /// Context class for entity framework + /// + /// The db context options + public RequestLoggerContext(DbContextOptions options) + : base(options) + { + } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(builder => + { + builder.HasKey(i => i.Id); + builder.HasAnnotation("Npgsql:ValueGenerationStrategy", + NpgsqlValueGenerationStrategy.IdentityByDefaultColumn); + builder.Property(p => p.Body).HasColumnType("jsonb"); + builder.Property(p => p.Headers).HasColumnType("jsonb"); + }); + } + + public virtual DbSet Requests { get; set; } = null!; } \ No newline at end of file diff --git a/src/RequestLogger.Tests/Unit/ProgramTests.cs b/src/RequestLogger.Tests/Unit/ProgramTests.cs index 2f35cc8..c829ec3 100644 --- a/src/RequestLogger.Tests/Unit/ProgramTests.cs +++ b/src/RequestLogger.Tests/Unit/ProgramTests.cs @@ -42,7 +42,7 @@ public ProgramTests() } [Fact] - public void Program_CallingGet_ReturnsResponse() + public async Task Program_CallingGet_ReturnsResponse() { // Arrange var request = _fixture.Create(); @@ -59,7 +59,7 @@ public void Program_CallingGet_ReturnsResponse() var client = app.CreateClient(); // Act - var response = client.GetAsync("/stuff/").Result; + var response = await client.GetAsync("/stuff/"); //Assert response.Should().NotBeNull(); @@ -68,7 +68,7 @@ public void Program_CallingGet_ReturnsResponse() } [Fact] - public void Program_MakingCall_ReturnsResponseFromDatabase() + public async Task Program_MakingCall_ReturnsResponseFromDatabase() { // Arrange var client = _sut.CreateClient(); @@ -76,107 +76,107 @@ public void Program_MakingCall_ReturnsResponseFromDatabase() // Act - var response = client.GetAsync("/stuff/").Result; + var response = await client.GetAsync("/stuff/"); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().BeNull(); + parsedContent?.Customer.Should().BeNull(); + parsedContent?.Body.Should().BeNull(); + parsedContent?.Verb.Should().Be("GET"); + parsedContent?.QueryParams.Should().BeNull(); } [Fact] - public void Program_MakingCall_ReturnsCustomerSuccessfully() + public async Task Program_MakingCall_ReturnsCustomerSuccessfully() { // Arrange var client = _sut.CreateClient(); // Act - var response = client.GetAsync("/stuff/customer/3425234").Result; + var response = await client.GetAsync("/stuff/customer/3425234"); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().Be("3425234"); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().BeNull(); + parsedContent?.Customer.Should().Be("3425234"); + parsedContent?.Body.Should().BeNull(); + parsedContent?.Verb.Should().Be("GET"); + parsedContent?.QueryParams.Should().BeNull(); } [Fact] - public void Program_MakingCall_ReturnsQueryParamsSuccessfully() + public async Task Program_MakingCall_ReturnsQueryParamsSuccessfully() { // Arrange var client = _sut.CreateClient(); // Act - var response = client.GetAsync("/some/url/3425234?query=something").Result; + var response = await client.GetAsync("/some/url/3425234?query=something"); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("GET"); - request?.QueryParams.Should().ContainKey("query"); + parsedContent?.Customer.Should().BeNull(); + parsedContent?.Body.Should().BeNull(); + parsedContent?.Verb.Should().Be("GET"); + parsedContent?.QueryParams.Should().ContainKey("query"); } [Fact] - public void Program_MakingPostCall_ReturnsCorrectVerb() + public async Task Program_MakingPostCall_ReturnsCorrectVerb() { // Arrange var client = _sut.CreateClient(); // Act - var response = client.PostAsync("/some/uri", new StringContent("")).Result; + var response = await client.PostAsync("/some/uri", new StringContent("")); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().BeNull(); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); + parsedContent?.Customer.Should().BeNull(); + parsedContent?.Body.Should().BeNull(); + parsedContent?.Verb.Should().Be("POST"); + parsedContent?.QueryParams.Should().BeNull(); } [Fact] - public void Program_MakingPostCallWithJsonBody_ReturnsCorrectBody() + public async Task Program_MakingPostCallWithJsonBody_ReturnsCorrectBody() { // Arrange var client = _sut.CreateClient(); var jsonBody = "{\"test\": \"test\"}"; // Act - var response = client.PostAsync("/some/uri", new StringContent(jsonBody)).Result; + var response = await client.PostAsync("/some/uri", new StringContent(jsonBody)); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().Be(jsonBody); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); + parsedContent?.Customer.Should().BeNull(); + parsedContent?.Body.Should().Be(jsonBody); + parsedContent?.Verb.Should().Be("POST"); + parsedContent?.QueryParams.Should().BeNull(); } [Fact] - public void Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() + public async Task Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() { // Arrange var client = _sut.CreateClient(); @@ -184,17 +184,17 @@ public void Program_MakingPostCallWithInvalidJsonBody_ReturnsCorrectBody() var jsonBody = "{ \"invalidJson\": \"test\" }"; // Act - var response = client.PostAsync("/some/uri", new StringContent(requestBody)).Result; + var response = await client.PostAsync("/some/uri", new StringContent(requestBody)); - var responseContent = response.Content.ReadAsStringAsync().Result; - var request = JsonSerializer.Deserialize(responseContent); + var responseContent = await response.Content.ReadAsStringAsync(); + var parsedContent = JsonSerializer.Deserialize(responseContent); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - request?.Customer.Should().BeNull(); - request?.Body.Should().Be(jsonBody); - request?.Verb.Should().Be("POST"); - request?.QueryParams.Should().BeNull(); + parsedContent?.Customer.Should().BeNull(); + parsedContent?.Body.Should().Be(jsonBody); + parsedContent?.Verb.Should().Be("POST"); + parsedContent?.QueryParams.Should().BeNull(); } } \ No newline at end of file diff --git a/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs b/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs index d1511f1..e88e1b7 100644 --- a/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs +++ b/src/RequestLogger.Tests/Unit/RequestLoggerServiceTests.cs @@ -1,64 +1,64 @@ -using AutoFixture; -using FluentAssertions; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Storage; -using Microsoft.Extensions.Options; -using Repository; -using RequestLogger.Dto; -using RequestLogger.Services; -using RequestLogger.Settings; - -namespace RequestLogger.Tests.Unit; - -public class RequestLoggerServiceTests -{ - private readonly RequestLoggerService _requestLoggerService; - - private readonly RequestLoggerContext _requestLoggerContext; - - private readonly Fixture _fixture; - - public RequestLoggerServiceTests() - { - var root = new InMemoryDatabaseRoot(); - - var options = new DbContextOptionsBuilder() - .UseInMemoryDatabase("test", root).Options; - _requestLoggerContext = new RequestLoggerContext(options); - - _requestLoggerService = new RequestLoggerService(_requestLoggerContext, new BlacklistService(Options.Create(new RequestLoggerSettings()))); - _fixture = new Fixture(); - } - - [Fact] - public void WriteLogMessage_ReturnsLogMessage_WhenCalledCorrectly() - { - // Arrange - var request = _fixture.Create(); - - // Act - var response = _requestLoggerService.WriteLogMessage(request).Result; - - //Assert - response?.Body.Should().Be(request.Body); - response?.Verb.Should().Be(request.Verb); - response?.Customer.Should().Be(request.Customer); - } - - [Fact] - public void WriteLogMessage_ActuallyWritesToDatabase_WhenCalledCorrectly() - { - // Arrange - var request = _fixture.Create(); - - // Act - _ = _requestLoggerService.WriteLogMessage(request).Result; - - //Assert - var databaseItem = _requestLoggerContext.Requests.FirstOrDefault(r => r.Id == 1); - - databaseItem?.Body.Should().Be(request.Body); - databaseItem?.Verb.Should().Be(request.Verb); - databaseItem?.Customer.Should().Be(request.Customer); - } +using AutoFixture; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; +using Microsoft.Extensions.Options; +using Repository; +using RequestLogger.Dto; +using RequestLogger.Services; +using RequestLogger.Settings; + +namespace RequestLogger.Tests.Unit; + +public class RequestLoggerServiceTests +{ + private readonly RequestLoggerService _requestLoggerService; + + private readonly RequestLoggerContext _requestLoggerContext; + + private readonly Fixture _fixture; + + public RequestLoggerServiceTests() + { + var root = new InMemoryDatabaseRoot(); + + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase("test", root).Options; + _requestLoggerContext = new RequestLoggerContext(options); + + _requestLoggerService = new RequestLoggerService(_requestLoggerContext, new BlacklistService(Options.Create(new RequestLoggerSettings()))); + _fixture = new Fixture(); + } + + [Fact] + public async Task WriteLogMessage_ReturnsLogMessage_WhenCalledCorrectly() + { + // Arrange + var request = _fixture.Create(); + + // Act + var response = await _requestLoggerService.WriteLogMessage(request); + + //Assert + response?.Body.Should().Be(request.Body); + response?.Verb.Should().Be(request.Verb); + response?.Customer.Should().Be(request.Customer); + } + + [Fact] + public async Task WriteLogMessage_ActuallyWritesToDatabase_WhenCalledCorrectly() + { + // Arrange + var request = _fixture.Create(); + + // Act + _ = await _requestLoggerService.WriteLogMessage(request); + + //Assert + var databaseItem = _requestLoggerContext.Requests.FirstOrDefault(r => r.Id == 1); + + databaseItem?.Body.Should().Be(request.Body); + databaseItem?.Verb.Should().Be(request.Verb); + databaseItem?.Customer.Should().Be(request.Customer); + } } \ No newline at end of file diff --git a/src/RequestLogger/Dto/Converters/RequestConverter.cs b/src/RequestLogger/Dto/Converters/RequestConverter.cs index 2e428d9..bf305ae 100644 --- a/src/RequestLogger/Dto/Converters/RequestConverter.cs +++ b/src/RequestLogger/Dto/Converters/RequestConverter.cs @@ -1,22 +1,22 @@ -using System.Text.Json; -using HttpRequest = Repository.Models.HttpRequest; - -namespace RequestLogger.Dto.Converters; - -public static class RequestConverter -{ - public static HttpRequest ConvertRequest(Request request) - { - return new HttpRequest - { - Verb = request.Verb, - Service = request.Service, - Customer = request.Customer, - Path = request.Path, - QueryParams = request.QueryParams != null ? JsonSerializer.Serialize(request.QueryParams) : null, - Body = request.Body, - Headers = JsonSerializer.Serialize(request.Headers), - RequestTime = request.RequestTime - }; - } +using System.Text.Json; +using HttpRequest = Repository.Models.HttpRequest; + +namespace RequestLogger.Dto.Converters; + +public static class RequestConverter +{ + public static HttpRequest ConvertRequest(Request request) + { + return new HttpRequest + { + Verb = request.Verb, + Service = request.Service, + Customer = request.Customer, + Path = request.Path, + QueryParams = request.QueryParams != null ? JsonSerializer.Serialize(request.QueryParams) : null, + Body = request.Body, + Headers = JsonSerializer.Serialize(request.Headers), + RequestTime = request.RequestTime + }; + } } \ No newline at end of file diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 15e4824..934707e 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -1,5 +1,7 @@ using System.Text.Json; using System.Web; +using HealthChecks.UI.Client; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; using Microsoft.EntityFrameworkCore; using Repository; using RequestLogger.Dto; @@ -19,19 +21,21 @@ Log.Logger = logger; -// Register Serilog builder.Logging.AddSerilog(logger); - -builder.Services.AddSingleton(); +builder.Services.AddHttpContextAccessor(); builder.Services.AddControllers(); -// Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle + builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); builder.Services.Configure(builder.Configuration.GetSection("RequestLoggerSettings")); builder.Services.AddRequestLoggerContext(builder.Configuration); + +builder.Services.AddHealthChecks().AddDbContextCheck(); + builder.Services.AddScoped(); + builder.Services.AddScoped(); var app = builder.Build(); @@ -47,42 +51,36 @@ RequestLoggerContextConfiguration.TryRunMigrations(builder.Configuration); +app.UseRouting(); + app.UseHttpsRedirection(); app.UseAuthorization(); +app.UseEndpoints(endpoints => +{ + endpoints.MapHealthChecks("/health", new HealthCheckOptions + { + ResponseWriter = UIResponseWriter.WriteHealthCheckUIResponse + }); +}); + app.Run(async (context) => { context.Request.EnableBuffering(); context.Request.Body.Position = 0; - var requestBody = await new StreamReader(context.Request.Body).ReadToEndAsync(); - - if (!requestBody.Equals(string.Empty)) - { - // just convert the body into minimal Json if it isn't Json - if (!IsJsonValid(requestBody)) - { - requestBody = $"{{ \"invalidJson\": \"{requestBody}\" }}"; - } - } - else - { - requestBody = null; - } + var requestBody = await GetRequestBody(context); using var scoped = app.Services.CreateScope(); - var loggerService = scoped.ServiceProvider.GetRequiredService(); // gets the customer id from a path like somePath/customer//somePath - var customerId = context.Request.Path.ToString().Split('/') - .SkipWhile(p => !p.Equals("customer", StringComparison.OrdinalIgnoreCase)).Skip(1).FirstOrDefault(); + var customerId = TryGetCustomerId(context); // converts query string into a dictionary (if it has values) - var parsedQueryString = HttpUtility.ParseQueryString(context.Request.QueryString.ToString()); - var queryStringDictionary = parsedQueryString.HasKeys() ? parsedQueryString.AllKeys.ToDictionary(k => k!, k => parsedQueryString[k]!) : null; + var queryStringDictionary = BuildQueryStringDictionary(context); - var service = context.Request.Headers.TryGetValue("X-Service", out var header) ? header.ToString() : context.Request.Host.Value; + var service = TryGetServiceName(context); var request = new Request() { @@ -96,11 +94,65 @@ RequestTime = DateTime.UtcNow }; - var requestCompleted = await loggerService.WriteLogMessage(request); - - await context.Response.WriteAsync(JsonSerializer.Serialize(requestCompleted)); + await SendResponse(request, context); }); +async Task SendResponse(Request request, HttpContext httpContext) +{ + try + { + await httpContext.Response.WriteAsync(JsonSerializer.Serialize(request)); + } + catch (Exception exception) + { + Log.Error(exception, "Error writing a response"); + } +} + +string TryGetServiceName(HttpContext httpContext) +{ + var s = httpContext.Request.Headers.TryGetValue("X-Service", out var header) + ? header.ToString() + : httpContext.Request.Host.Value; + return s; +} + +Dictionary? BuildQueryStringDictionary(HttpContext httpContext) +{ + var parsedQueryString = HttpUtility.ParseQueryString(httpContext.Request.QueryString.ToString()); + var dictionary = parsedQueryString.HasKeys() + ? parsedQueryString.AllKeys.ToDictionary(k => k!, k => parsedQueryString[k]!) + : null; + return dictionary; +} + +string? TryGetCustomerId(HttpContext httpContext) +{ + var s = httpContext.Request.Path.ToString().Split('/') + .SkipWhile(p => !p.Equals("customer", StringComparison.OrdinalIgnoreCase)).Skip(1).FirstOrDefault(); + return s; +} + +async Task GetRequestBody(HttpContext context) +{ + var requestBody = await new StreamReader(context.Request.Body).ReadToEndAsync(); + + if (!requestBody.Equals(string.Empty)) + { + // just convert the body into minimal Json if it isn't Json + if (!IsJsonValid(requestBody)) + { + requestBody = $"{{ \"invalidJson\": \"{requestBody}\" }}"; + } + } + else + { + requestBody = null; + } + + return requestBody; +} + bool IsJsonValid(string json) { if (string.IsNullOrWhiteSpace(json)) diff --git a/src/RequestLogger/RequestLogger.csproj b/src/RequestLogger/RequestLogger.csproj index 8182311..d5d944b 100644 --- a/src/RequestLogger/RequestLogger.csproj +++ b/src/RequestLogger/RequestLogger.csproj @@ -8,6 +8,8 @@ + + @@ -16,6 +18,7 @@ + From 7db110e6cb33ee1fb086cc3e4285c624afbd1be0 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 11 Jul 2023 13:53:42 +0100 Subject: [PATCH 22/25] fixing broken test --- src/RequestLogger.Tests/Unit/ProgramTests.cs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/RequestLogger.Tests/Unit/ProgramTests.cs b/src/RequestLogger.Tests/Unit/ProgramTests.cs index c829ec3..6297b75 100644 --- a/src/RequestLogger.Tests/Unit/ProgramTests.cs +++ b/src/RequestLogger.Tests/Unit/ProgramTests.cs @@ -19,7 +19,6 @@ namespace RequestLogger.Tests.Unit; public class ProgramTests { private readonly WebApplicationFactory _sut; - private readonly Fixture _fixture; public ProgramTests() { @@ -37,34 +36,22 @@ public ProgramTests() services.AddScoped(); services.AddDbContext(x => x.UseInMemoryDatabase("Testing", root)); })); - - _fixture = new Fixture(); } [Fact] public async Task Program_CallingGet_ReturnsResponse() { // Arrange - var request = _fixture.Create(); - - var fakeRequestLoggerService = A.Fake(); - A.CallTo(() => fakeRequestLoggerService.WriteLogMessage(A.Ignored)).Returns(Task.FromResult(request)); - - var app = new WebApplicationFactory() - .WithWebHostBuilder(builder => builder.ConfigureServices(services => - { - services.AddScoped(_ => fakeRequestLoggerService); - })); - - var client = app.CreateClient(); + var client = _sut.CreateClient(); // Act var response = await client.GetAsync("/stuff/"); + var responseBody = await response.Content.ReadAsStringAsync(); //Assert response.Should().NotBeNull(); response.IsSuccessStatusCode.Should().BeTrue(); - response.Content.ReadAsStringAsync().Result.Should().Contain(request.Verb); + responseBody.Should().Contain("GET"); } [Fact] From d19126e9801de20057aab49ea12cbb2415c333e5 Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 11 Jul 2023 15:05:45 +0100 Subject: [PATCH 23/25] changing customer to customers --- src/RequestLogger/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 934707e..9388b12 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -129,7 +129,7 @@ string TryGetServiceName(HttpContext httpContext) string? TryGetCustomerId(HttpContext httpContext) { var s = httpContext.Request.Path.ToString().Split('/') - .SkipWhile(p => !p.Equals("customer", StringComparison.OrdinalIgnoreCase)).Skip(1).FirstOrDefault(); + .SkipWhile(p => !p.Equals("customers", StringComparison.OrdinalIgnoreCase)).Skip(1).FirstOrDefault(); return s; } From e18c09e9a7b2e4e45dcb3fc39e8a38bfa9477c8c Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 11 Jul 2023 15:16:44 +0100 Subject: [PATCH 24/25] pushing up fixed test --- src/RequestLogger.Tests/Unit/ProgramTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RequestLogger.Tests/Unit/ProgramTests.cs b/src/RequestLogger.Tests/Unit/ProgramTests.cs index 6297b75..1ba8678 100644 --- a/src/RequestLogger.Tests/Unit/ProgramTests.cs +++ b/src/RequestLogger.Tests/Unit/ProgramTests.cs @@ -84,7 +84,7 @@ public async Task Program_MakingCall_ReturnsCustomerSuccessfully() var client = _sut.CreateClient(); // Act - var response = await client.GetAsync("/stuff/customer/3425234"); + var response = await client.GetAsync("/stuff/customers/3425234"); var responseContent = await response.Content.ReadAsStringAsync(); var parsedContent = JsonSerializer.Deserialize(responseContent); From ba1bb5d3196e882675c202af444891dbf003fa2e Mon Sep 17 00:00:00 2001 From: "jack.lewis" Date: Tue, 11 Jul 2023 15:29:39 +0100 Subject: [PATCH 25/25] adding back in removed logger --- src/RequestLogger/Program.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/RequestLogger/Program.cs b/src/RequestLogger/Program.cs index 9388b12..8840146 100644 --- a/src/RequestLogger/Program.cs +++ b/src/RequestLogger/Program.cs @@ -76,6 +76,8 @@ // gets the customer id from a path like somePath/customer//somePath var customerId = TryGetCustomerId(context); + + var requestLoggerService = scoped.ServiceProvider.GetRequiredService(); // converts query string into a dictionary (if it has values) var queryStringDictionary = BuildQueryStringDictionary(context); @@ -94,10 +96,12 @@ RequestTime = DateTime.UtcNow }; - await SendResponse(request, context); + var requestCompleted = await requestLoggerService.WriteLogMessage(request); + + await SendResponse(requestCompleted, context); }); -async Task SendResponse(Request request, HttpContext httpContext) +async Task SendResponse(Request? request, HttpContext httpContext) { try {