diff --git a/Project.toml b/Project.toml index 9c673ae..227df78 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "ConfigEnv" uuid = "01234589-554d-41b7-ae5c-7b6ee2db6796" authors = ["Andrey Oskin "] -version = "0.2.0" +version = "0.2.1" [deps] diff --git a/src/ConfigEnv.jl b/src/ConfigEnv.jl index 30bdbcd..1485530 100644 --- a/src/ConfigEnv.jl +++ b/src/ConfigEnv.jl @@ -2,24 +2,238 @@ module ConfigEnv import Base: getindex, get, isempty, merge, merge!, haskey -export dotenv, dotenvx +export dotenv, dotenvx, isresolved, unresolved_keys -struct EnvProxyDict +struct EnvProxyDict{T} dict::Dict{String, String} + env::T + undefined::Vector{String} + circular::Vector{String} end +EnvProxyDict(dict) = EnvProxyDict(dict, ENV, String[], String[]) +EnvProxyDict(dict, env) = EnvProxyDict(dict, env, String[], String[]) -getindex(ed::EnvProxyDict, key) = get(ed.dict, key, ENV[key]) -get(ed::EnvProxyDict, key, default) = get(ed.dict, key, get(ENV, key, default)) +getindex(ed::EnvProxyDict, key) = get(ed.dict, key, ed.env[key]) +get(ed::EnvProxyDict, key, default) = get(ed.dict, key, get(ed.env, key, default)) isempty(ed::EnvProxyDict) = isempty(ed.dict) -merge(epd1::EnvProxyDict, epd2::EnvProxyDict...) = EnvProxyDict(foldl((x, y) -> merge(x, y.dict), epd2; init = epd1.dict)) -haskey(ed::EnvProxyDict, key) = haskey(ed.dict, key) || haskey(ENV, key) +Base.:*(epd1::EnvProxyDict, epd2::EnvProxyDict...) = merge(epd1, epd2...; overwrite = false) +function imprint(epd::EnvProxyDict, env) + for (k, v) in epd.dict + env[k] = v + end +end + +function merge(epd1::EnvProxyDict, epd2::EnvProxyDict...; overwrite = true) + epd = EnvProxyDict(foldl((x, y) -> merge(x, y.dict), epd2; init = epd1.dict), epd1.env) + resolve!(epd, epd.env) + overwrite && imprint(epd, epd.env) + epd +end + +haskey(ed::EnvProxyDict, key) = haskey(ed.dict, key) || haskey(ed.env, key) -function merge!(epd1::EnvProxyDict, epd2::EnvProxyDict...) +function merge!(epd1::EnvProxyDict, epd2::EnvProxyDict...; overwrite = true) foldl((x, y) -> merge!(x, y.dict), epd2; init = epd1.dict) - epd1 + # You shouldn't use different envs during `merge!`. This is undefined behaviour. + # first dictionary environment can't be changed, cause it can be `ENV` + resolve!(epd1, epd1.env) + overwrite && imprint(epd1, epd1.env) + return epd1 +end + +function isdefin(s, i = 0, lev = 0) + i = nextind(s, i) + len = ncodeunits(s) + mode = 0 + cnt_brackets = 0 + while i <= len + c = s[i] + if c == '$' + mode = 1 + elseif c == '{' + if mode == 1 + return isdefin(s, i, 1) + else + cnt_brackets += 1 + mode = 0 + end + elseif c == '}' + mode = 0 + cnt_brackets -= 1 + lev == 1 && cnt_brackets < 0 && return false + else + mode = 0 + end + i = nextind(s, i) + end + + return true +end + +mutable struct KVNode + key::String + value::String + children::Vector{KVNode} + parents_cnt::Int + isfinal::Bool + isresolved::Bool +end + +KVNode(k, v) = KVNode(k, v, KVNode[], 0, true, false) +KVNode(v) = KVNode("", v, KVNode[], 0, false, false) +isresolved(kvnode::KVNode) = kvnode.isresolved +isresolved(epd::EnvProxyDict) = isempty(epd.undefined) && isempty(epd.circular) + +function unresolved_keys(edp::EnvProxyDict) + undefined = map(k -> k => edp.dict[k], edp.undefined) + circular = map(k -> k => edp.dict[k], edp.circular) + + return (; circular = circular, undefined = undefined) +end + +function destructure!(v::KVNode, stack, knodes, i = 0, env = ENV) + val = v.value + len = ncodeunits(val) + i0 = nextind(val, i) + i = i0 + bracket_count = 0 + mode = 0 # everything is normal + isvalid = v.isfinal ? true : false + + while i <= len + c = val[i] + if c == '$' + mode = 1 # waiting for the opening { + elseif c == '{' + if mode == 1 + node = KVNode(val) + push!(node.children, v) + i, valid = destructure!(node, stack, knodes, i, env) # should return index of the closing bracket } + if valid + v.parents_cnt += 1 + isvalid = true + end + else + bracket_count += 1 + end + mode = 0 # calm down + elseif c == '}' # we should produce some reasonable result if this is our closing bracket + bracket_count -= 1 + mode = 0 + if !v.isfinal + bracket_count >= 0 && continue # false alarm + if v.parents_cnt == 0 # we are key node + v.key = v.value[i0:i-1] + if haskey(knodes, v.key) + append!(knodes[v.key].children, v.children) + empty!(v.children) + elseif haskey(env, v.key) + v.value = env[v.key] + push!(stack, v) + else + # Unknown key + v.value = v.value[i0-2:i] + push!(stack, v) + end + else # we are intermidiate node + v.value = v.value[i0:i-1] + end + return i, true + end + else + mode = 0 + end + + if i <= len + i = nextind(val, i) + end + end + return i, isvalid +end + +function resolve!(edp::EnvProxyDict, env = ENV) + knodes = Dict{String, KVNode}() + for (k, v) in edp.dict + knodes[k] = KVNode(k, v) + end + stack = KVNode[] + + prepare_stack!(stack, knodes, env) + + resolve!(stack, knodes, env) + + empty!(edp.undefined) + empty!(edp.circular) + for (k, v) in knodes + edp.dict[k] = v.value + if !v.isresolved + push!(edp.circular, k) + elseif !isdefin(v.value) + push!(edp.undefined, k) + end + end + + edp +end + +function prepare_stack!(stack, knodes, env) + for v in values(knodes) + # we extract interpolated terms from the value and last tier should be put on + # the stack for further imprinting + destructure!(v, stack, knodes, 0, env) + end + + for v in values(knodes) + # if we do not have anything to interpolate inside the value, we are good. + # If we still have some other value where node should be used for interpolation, + # we put node on the interpolation stack + if v.parents_cnt == 0 + v.isresolved = true + if !isempty(v.children) + push!(stack, v) + end + end + end +end + +function recursive_replace!(node, key, val) + occursin(key, node.value) || return + node.value = replace(node.value, key => val) + foreach(x -> recursive_replace!(x, key, val), node.children) + nothing end -Base.:*(epd1::EnvProxyDict, epd2::EnvProxyDict...) = EnvProxyDict(foldl((x, y) -> merge(x, y.dict), epd2; init = epd1.dict)) +function resolve!(stack, knodes, env) + while !isempty(stack) + v = pop!(stack) + key = "\${" * v.key * "}" + while !isempty(v.children) + kid = pop!(v.children) + recursive_replace!(kid, key, v.value) + kid.parents_cnt -= 1 + kid.parents_cnt == 0 || continue + if kid.isfinal + kid.isresolved = true + else + kid.key = kid.value + if haskey(knodes, kid.key) + append!(knodes[kid.key].children, kid.children) + empty!(kid.children) + if !isempty(knodes[kid.key].children) && knodes[kid.key].parents_cnt == 0 + push!(stack, knodes[kid.key]) + end + elseif haskey(env, kid.key) + kid.value = env[kid.key] + else + kid.value = "\${" * kid.key * "}" + end + end + if !isempty(kid.children) + push!(stack, kid) + end + end + end +end """ `ConfigEnv.parse` accepts a String or an IOBuffer (any value that @@ -89,23 +303,26 @@ julia> cfg["USER"] john_doe ``` """ -function dotenv(path = ".env"; overwrite = true) +function dotenv(path = ".env"; overwrite = true, env = ENV) parsed = if (validatefile(path)) parse(read(path, String)) else parse(path) end - for (k, v) in parsed - if !haskey(ENV, k) || overwrite - ENV[k] = v + epd = EnvProxyDict(parsed, env) + resolve!(epd, env) + + for (k, v) in epd.dict + if !haskey(env, k) || overwrite + env[k] = v end end - return EnvProxyDict(parsed) + return epd end -dotenv(paths...; overwrite = true) = merge!(dotenv.(paths; overwrite = overwrite)...) +dotenv(paths...; overwrite = true, env = ENV) = merge!(dotenv.(paths; overwrite = overwrite, env = env)..., overwrite = overwrite) """ dotenvx(path1, path2, ...; overwrite = false) @@ -138,6 +355,6 @@ julia> cfg["USER"] john_doe ``` """ -dotenvx(paths...; overwrite = false) = dotenv(paths...; overwrite = overwrite) +dotenvx(paths...; overwrite = false, env = ENV) = dotenv(paths...; overwrite = overwrite, env = env) end diff --git a/test/.env_interpolation1 b/test/.env_interpolation1 new file mode 100644 index 0000000..a04416f --- /dev/null +++ b/test/.env_interpolation1 @@ -0,0 +1,2 @@ +X = A_${Y} +Y = FOO diff --git a/test/.env_interpolation2 b/test/.env_interpolation2 new file mode 100644 index 0000000..965e951 --- /dev/null +++ b/test/.env_interpolation2 @@ -0,0 +1,3 @@ +USER = ${USER_${N}} +N = 1 +USER_1 = FOO diff --git a/test/.env_interpolation3 b/test/.env_interpolation3 new file mode 100644 index 0000000..5d1bc02 --- /dev/null +++ b/test/.env_interpolation3 @@ -0,0 +1,2 @@ +USER = ${USER_${N}} +USER_1 = FOO diff --git a/test/.env_interpolation4 b/test/.env_interpolation4 new file mode 100644 index 0000000..257d35a --- /dev/null +++ b/test/.env_interpolation4 @@ -0,0 +1 @@ +N = 1 diff --git a/test/.env_interpolation5 b/test/.env_interpolation5 new file mode 100644 index 0000000..c32cf34 --- /dev/null +++ b/test/.env_interpolation5 @@ -0,0 +1,2 @@ +X = ${Y} +Y = ${X} diff --git a/test/test04_interpolations.jl b/test/test04_interpolations.jl new file mode 100644 index 0000000..ee3515f --- /dev/null +++ b/test/test04_interpolations.jl @@ -0,0 +1,319 @@ +module TestInterpolations + +using ConfigEnv +using ConfigEnv: KVNode, resolve!, prepare_stack!, isdefin +using Test + +const dir = dirname(@__FILE__) + +@testset "Internal implementation" begin + @testset "Simple interpolation" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${Y}"), + "Y" => KVNode("Y", "Z")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "Z" + + @test knodes["Y"].isresolved + @test knodes["Y"].value == "Z" + end + + @testset "Simple interpolation with extra characters" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "FOO_\${Y}"), + "Y" => KVNode("Y", "Z")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "FOO_Z" + + @test knodes["Y"].isresolved + @test knodes["Y"].value == "Z" + end + + @testset "Circular dependencies" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${Y}"), + "Y" => KVNode("Y", "\${X}")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test !knodes["X"].isresolved + @test knodes["X"].value == "\${Y}" + + @test !knodes["Y"].isresolved + @test knodes["Y"].value == "\${X}" + end + + @testset "Two level interpolation" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "A_\${Y}"), + "Y" => KVNode("Y", "B_\${Z}"), + "Z" => KVNode("Z", "FOO")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "A_B_FOO" + + @test knodes["Y"].isresolved + @test knodes["Y"].value == "B_FOO" + + @test knodes["Z"].isresolved + @test knodes["Z"].value == "FOO" + end + + @testset "Nested interpolation" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${USER_\${N}}"), + "N" => KVNode("N", "1"), + "USER_1" => KVNode("USER_1", "FOO")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "FOO" + + @test knodes["N"].isresolved + @test knodes["N"].value == "1" + + @test knodes["USER_1"].isresolved + @test knodes["USER_1"].value == "FOO" + end + + @testset "Two neighbours interpolation" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${FOO}_\${BAR}"), + "FOO" => KVNode("FOO", "A"), + "BAR" => KVNode("BAR", "B")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "A_B" + + @test knodes["FOO"].isresolved + @test knodes["FOO"].value == "A" + + @test knodes["BAR"].isresolved + @test knodes["BAR"].value == "B" + end + + @testset "One unknown key" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${FOO}_\${BAR}"), + "FOO" => KVNode("FOO", "A")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "A_\${BAR}" + + @test knodes["FOO"].isresolved + @test knodes["FOO"].value == "A" + end + + @testset "One key from ENV" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${FOO}_\${BAR}"), + "FOO" => KVNode("FOO", "A")) + env = Dict{String, String}("BAR" => "zzz") + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "A_zzz" + + @test knodes["FOO"].isresolved + @test knodes["FOO"].value == "A" + end + + @testset "Three level circular dependencies" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${Y}"), + "Y" => KVNode("Y", "\${Z}"), + "Z" => KVNode("Z", "\${X}")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test !knodes["X"].isresolved + @test knodes["X"].value == "\${Y}" + + @test !knodes["Y"].isresolved + @test knodes["Y"].value == "\${Z}" + + @test !knodes["Z"].isresolved + @test knodes["Z"].value == "\${X}" + end + + @testset "Broken key" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "\${Y"), + "Y" => KVNode("Y", "FOO")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "\${Y" + + @test knodes["Y"].isresolved + @test knodes["Y"].value == "FOO" + end + + @testset "Json substring is ignored" begin + stack = KVNode[] + knodes = Dict("X" => KVNode("X", "A_\${Y}"), + "Y" => KVNode("Y", "{foo: bar}")) + env = Dict{String, String}() + + prepare_stack!(stack, knodes, env) + resolve!(stack, knodes, env) + + @test knodes["X"].isresolved + @test knodes["X"].value == "A_{foo: bar}" + + @test knodes["Y"].isresolved + @test knodes["Y"].value == "{foo: bar}" + end +end + +@testset "Test isdefin" begin + @test isdefin("asd") + @test isdefin("zxc\${asdf") + @test isdefin("zxc\${as\${df") + @test isdefin("zxc\${") + + @test !isdefin("asd\${xzcvcv}") + @test !isdefin("asd\${xzcvcv}cxv") + @test !isdefin("asd\${xz\${cv}cv}") +end + +@testset "File interpolations" begin + @testset "Simple file interpolation" begin + haskey(ENV, "X") && pop!(ENV, "X") + haskey(ENV, "Y") && pop!(ENV, "Y") + file = joinpath(dir, ".env_interpolation1") + cfg = dotenv(file) + + @test isresolved(cfg) + @test isempty(unresolved_keys(cfg).circular) + @test isempty(unresolved_keys(cfg).undefined) + @test cfg["X"] == "A_FOO" + @test cfg["Y"] == "FOO" + end + + @testset "Nested file interpolation" begin + haskey(ENV, "USER") && pop!(ENV, "USER") + haskey(ENV, "N") && pop!(ENV, "N") + haskey(ENV, "USER_1") && pop!(ENV, "USER_1") + file = joinpath(dir, ".env_interpolation2") + cfg = dotenv(file) + + @test isresolved(cfg) + @test isempty(unresolved_keys(cfg).circular) + @test isempty(unresolved_keys(cfg).undefined) + @test cfg["USER"] == "FOO" + @test cfg["N"] == "1" + @test cfg["USER_1"] == "FOO" + end + + @testset "Merge file interpolation" begin + haskey(ENV, "USER") && pop!(ENV, "USER") + haskey(ENV, "N") && pop!(ENV, "N") + haskey(ENV, "USER_1") && pop!(ENV, "USER_1") + file1 = joinpath(dir, ".env_interpolation3") + file2 = joinpath(dir, ".env_interpolation4") + + cfg = dotenv(file1, file2) + + @test isresolved(cfg) + @test isempty(unresolved_keys(cfg).circular) + @test isempty(unresolved_keys(cfg).undefined) + @test cfg["USER"] == "FOO" + @test cfg["N"] == "1" + @test cfg["USER_1"] == "FOO" + + haskey(ENV, "USER") && pop!(ENV, "USER") + haskey(ENV, "N") && pop!(ENV, "N") + haskey(ENV, "USER_1") && pop!(ENV, "USER_1") + cfg1 = dotenv(file1) + @test !isresolved(cfg1) + @test isempty(unresolved_keys(cfg1).circular) + @test !isempty(unresolved_keys(cfg1).undefined) + cfg2 = dotenv(file2) + cfg = cfg1 * cfg2 + @test isresolved(cfg) + @test isempty(unresolved_keys(cfg).circular) + @test isempty(unresolved_keys(cfg).undefined) + @test cfg["USER"] == "FOO" + @test cfg["N"] == "1" + @test cfg["USER_1"] == "FOO" + + haskey(ENV, "USER") && pop!(ENV, "USER") + haskey(ENV, "N") && pop!(ENV, "N") + haskey(ENV, "USER_1") && pop!(ENV, "USER_1") + cfg1 = dotenv(file1) + cfg2 = dotenv(file2) + cfg = merge(cfg1, cfg2) + @test isresolved(cfg) + @test isempty(unresolved_keys(cfg).circular) + @test isempty(unresolved_keys(cfg).undefined) + @test cfg["USER"] == "FOO" + @test cfg["N"] == "1" + @test cfg["USER_1"] == "FOO" + + haskey(ENV, "USER") && pop!(ENV, "USER") + haskey(ENV, "N") && pop!(ENV, "N") + haskey(ENV, "USER_1") && pop!(ENV, "USER_1") + cfg1 = dotenv(file1) + cfg2 = dotenv(file2) + merge!(cfg1, cfg2) + @test isresolved(cfg1) + @test isempty(unresolved_keys(cfg1).circular) + @test isempty(unresolved_keys(cfg1).undefined) + @test cfg1["USER"] == "FOO" + @test cfg1["N"] == "1" + @test cfg1["USER_1"] == "FOO" + end + + @testset "Circular file interpolation" begin + haskey(ENV, "X") && pop!(ENV, "X") + haskey(ENV, "Y") && pop!(ENV, "Y") + file = joinpath(dir, ".env_interpolation5") + cfg = dotenv(file) + + @test !isresolved(cfg) + badkeys = unresolved_keys(cfg).circular + sort!(badkeys, by = x -> x[1]) + @test length(badkeys) == 2 + @test badkeys[1][2] == "\${Y}" + @test badkeys[2][2] == "\${X}" + @test cfg["X"] == "\${Y}" + @test cfg["Y"] == "\${X}" + end +end + +end # module