Major update

scripts/mirabelle/empty2null.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# usage: ./empty2null.sh mydb.db
+
+teefile=tmp.txt
+
+function empty2null() {
+  for t in $(
+sqlite3 "$db" "SELECT name FROM sqlite_master
+WHERE type=='table' ORDER BY name" | tee ${teefile} )
+  do
+    nc=0
+    onec=''
+    mulc=''
+    where='WHERE'
+    cma="UPDATE OR ABORT \"${t}\" SET
+ "
+    for c in $( 
+printf "SELECT '\"'||name||'\"' FROM pragma_table_info('%s')
+WHERE type == 'INTEGER' or type == 'FLOAT' ORDER BY cid ASC;\\n" "$t" \
+      | sqlite3 "$db" | tee -a ${teefile} )
+    do
+      onec+=$(printf "%s %s = NULL" "$cma" "$c")
+      mulc+=$(printf "%s %s = IIF(%s=='',NULL,%s)" "$cma" "$c" "$c" "$c")
+      cma='
+,'
+      test $nc -gt 0 && where+=' OR'
+      where+=" $c==''"
+      nc=$(( nc + 1 ))
+    done
+    if [ $nc -gt 0 ]
+    then
+      if [ $nc -gt 1 ]
+      then
+        printf '%s\n  %s;\n' "$mulc" "${where}"
+      else
+        printf '%s\n  %s;\n' "$onec" "${where}"
+      fi
+    fi
+  done
+}
+
+db=$1
+
+echo "$(date +'%Y-%m-%dT%H:%M:%S') - Building query"
+sql=$(empty2null)
+echo "$sql"
+
+echo "$(date +'%Y-%m-%dT%H:%M:%S') - Converting..."
+sqlite3 "$db" "$sql"
+echo "$(date +'%Y-%m-%dT%H:%M:%S') - ... converting empty to NULL ended"

scripts/mirabelle/products.schema.2024-01-18

@@ -0,0 +1,210 @@
+/* head -n 1 en.openfoodfacts.org.products.csv | /usr/local/bin/vd -f tsv */;
+/* head -n 1 en.openfoodfacts.org.products.csv | tr "\t" "\n" | wc -l */;
+CREATE TABLE IF NOT EXISTS "all" (
+   [code] TEXT,
+   [url] TEXT,
+   [creator] TEXT,
+   [created_t] INTEGER,
+   [created_datetime] TEXT,
+   [last_modified_t] INTEGER,
+   [last_modified_datetime] TEXT,
+   [last_modified_by] TEXT,
+   [last_updated_t] INTEGER,
+   [last_updated_datetime] TEXT,
+   [product_name] TEXT,
+   [abbreviated_product_name] TEXT,
+   [generic_name] TEXT,
+   [quantity] TEXT,
+   [packaging] TEXT,
+   [packaging_tags] TEXT,
+   [packaging_en] TEXT,
+   [packaging_text] TEXT,
+   [brands] TEXT,
+   [brands_tags] TEXT,
+   [categories] TEXT,
+   [categories_tags] TEXT,
+   [categories_en] TEXT,
+   [origins] TEXT,
+   [origins_tags] TEXT,
+   [origins_en] TEXT,
+   [manufacturing_places] TEXT,
+   [manufacturing_places_tags] TEXT,
+   [labels] TEXT,
+   [labels_tags] TEXT,
+   [labels_en] TEXT,
+   [emb_codes] TEXT,
+   [emb_codes_tags] TEXT,
+   [first_packaging_code_geo] TEXT,
+   [cities] TEXT,
+   [cities_tags] TEXT,
+   [purchase_places] TEXT,
+   [stores] TEXT,
+   [countries] TEXT,
+   [countries_tags] TEXT,
+   [countries_en] TEXT,
+   [ingredients_text] TEXT,
+   [ingredients_tags] TEXT,
+   [ingredients_analysis_tags] TEXT,
+   [allergens] TEXT,
+   [allergens_en] TEXT,
+   [traces] TEXT,
+   [traces_tags] TEXT,
+   [traces_en] TEXT,
+   [serving_size] TEXT,
+   [serving_quantity] FLOAT,
+   [no_nutrition_data] INTEGER,
+   [additives_n] INTEGER,
+   [additives] TEXT,
+   [additives_tags] TEXT,
+   [additives_en] TEXT,
+   [nutriscore_score] INTEGER,
+   [nutriscore_grade] TEXT,
+   [nova_group] INTEGER,
+   [pnns_groups_1] TEXT,
+   [pnns_groups_2] TEXT,
+   [food_groups] TEXT,
+   [food_groups_tags] TEXT,
+   [food_groups_en] TEXT,
+   [states] TEXT,
+   [states_tags] TEXT,
+   [states_en] TEXT,
+   [brand_owner] TEXT,
+   [ecoscore_score] FLOAT,
+   [ecoscore_grade] TEXT,
+   [nutrient_levels_tags] TEXT,
+   [product_quantity] TEXT,
+   [owner] TEXT,
+   [data_quality_errors_tags] TEXT,
+   [unique_scans_n] INTEGER,
+   [popularity_tags] TEXT,
+   [completeness] FLOAT,
+   [last_image_t] INTEGER,
+   [last_image_datetime] TEXT,
+   [main_category] TEXT,
+   [main_category_en] TEXT,
+   [image_url] TEXT,
+   [image_small_url] TEXT,
+   [image_ingredients_url] TEXT,
+   [image_ingredients_small_url] TEXT,
+   [image_nutrition_url] TEXT,
+   [image_nutrition_small_url] TEXT,
+   [energy-kj_100g] FLOAT,
+   [energy-kcal_100g] FLOAT,
+   [energy_100g] FLOAT,
+   [energy-from-fat_100g] FLOAT,
+   [fat_100g] FLOAT,
+   [saturated-fat_100g] FLOAT,
+   [butyric-acid_100g] FLOAT,
+   [caproic-acid_100g] FLOAT,
+   [caprylic-acid_100g] FLOAT,
+   [capric-acid_100g] FLOAT,
+   [lauric-acid_100g] FLOAT,
+   [myristic-acid_100g] FLOAT,
+   [palmitic-acid_100g] FLOAT,
+   [stearic-acid_100g] FLOAT,
+   [arachidic-acid_100g] FLOAT,
+   [behenic-acid_100g] FLOAT,
+   [lignoceric-acid_100g] FLOAT,
+   [cerotic-acid_100g] FLOAT,
+   [montanic-acid_100g] FLOAT,
+   [melissic-acid_100g] FLOAT,
+   [unsaturated-fat_100g] FLOAT,
+   [monounsaturated-fat_100g] FLOAT,
+   [omega-9-fat_100g] FLOAT,
+   [polyunsaturated-fat_100g] FLOAT,
+   [omega-3-fat_100g] FLOAT,
+   [omega-6-fat_100g] FLOAT,
+   [alpha-linolenic-acid_100g] FLOAT,
+   [eicosapentaenoic-acid_100g] FLOAT,
+   [docosahexaenoic-acid_100g] FLOAT,
+   [linoleic-acid_100g] FLOAT,
+   [arachidonic-acid_100g] FLOAT,
+   [gamma-linolenic-acid_100g] FLOAT,
+   [dihomo-gamma-linolenic-acid_100g] FLOAT,
+   [oleic-acid_100g] FLOAT,
+   [elaidic-acid_100g] FLOAT,
+   [gondoic-acid_100g] FLOAT,
+   [mead-acid_100g] FLOAT,
+   [erucic-acid_100g] FLOAT,
+   [nervonic-acid_100g] FLOAT,
+   [trans-fat_100g] FLOAT,
+   [cholesterol_100g] FLOAT,
+   [carbohydrates_100g] FLOAT,
+   [sugars_100g] FLOAT,
+   [added-sugars_100g] FLOAT,
+   [sucrose_100g] FLOAT,
+   [glucose_100g] FLOAT,
+   [fructose_100g] FLOAT,
+   [lactose_100g] FLOAT,
+   [maltose_100g] FLOAT,
+   [maltodextrins_100g] FLOAT,
+   [starch_100g] FLOAT,
+   [polyols_100g] FLOAT,
+   [erythritol_100g] FLOAT,
+   [fiber_100g] FLOAT,
+   [soluble-fiber_100g] FLOAT,
+   [insoluble-fiber_100g] FLOAT,
+   [proteins_100g] FLOAT,
+   [casein_100g] FLOAT,
+   [serum-proteins_100g] FLOAT,
+   [nucleotides_100g] FLOAT,
+   [salt_100g] FLOAT,
+   [added-salt_100g] FLOAT,
+   [sodium_100g] FLOAT,
+   [alcohol_100g] FLOAT,
+   [vitamin-a_100g] FLOAT,
+   [beta-carotene_100g] FLOAT,
+   [vitamin-d_100g] FLOAT,
+   [vitamin-e_100g] FLOAT,
+   [vitamin-k_100g] FLOAT,
+   [vitamin-c_100g] FLOAT,
+   [vitamin-b1_100g] FLOAT,
+   [vitamin-b2_100g] FLOAT,
+   [vitamin-pp_100g] FLOAT,
+   [vitamin-b6_100g] FLOAT,
+   [vitamin-b9_100g] FLOAT,
+   [folates_100g] FLOAT,
+   [vitamin-b12_100g] FLOAT,
+   [biotin_100g] FLOAT,
+   [pantothenic-acid_100g] FLOAT,
+   [silica_100g] FLOAT,
+   [bicarbonate_100g] FLOAT,
+   [potassium_100g] FLOAT,
+   [chloride_100g] FLOAT,
+   [calcium_100g] FLOAT,
+   [phosphorus_100g] FLOAT,
+   [iron_100g] FLOAT,
+   [magnesium_100g] FLOAT,
+   [zinc_100g] FLOAT,
+   [copper_100g] FLOAT,
+   [manganese_100g] FLOAT,
+   [fluoride_100g] FLOAT,
+   [selenium_100g] FLOAT,
+   [chromium_100g] FLOAT,
+   [molybdenum_100g] FLOAT,
+   [iodine_100g] FLOAT,
+   [caffeine_100g] FLOAT,
+   [taurine_100g] FLOAT,
+   [ph_100g] FLOAT,
+   [fruits-vegetables-nuts_100g] FLOAT,
+   [fruits-vegetables-nuts-dried_100g] FLOAT,
+   [fruits-vegetables-nuts-estimate_100g] FLOAT,
+   [fruits-vegetables-nuts-estimate-from-ingredients_100g] FLOAT,
+   [collagen-meat-protein-ratio_100g] FLOAT,
+   [cocoa_100g] FLOAT,
+   [chlorophyl_100g] FLOAT,
+   [carbon-footprint_100g] FLOAT,
+   [carbon-footprint-from-meat-or-fish_100g] FLOAT,
+   [nutrition-score-fr_100g] FLOAT,
+   [nutrition-score-uk_100g] FLOAT,
+   [glycemic-index_100g] FLOAT,
+   [water-hardness_100g] FLOAT,
+   [choline_100g] FLOAT,
+   [phylloquinone_100g] FLOAT,
+   [beta-glucan_100g] FLOAT,
+   [inositol_100g] FLOAT,
+   [carnitine_100g] FLOAT,
+   [sulphate_100g] FLOAT,
+   [nitrate_100g] FLOAT,
+   [acidity_100g] FLOAT
+);