Development

.github/workflows/build_base_container.yml

@@ -16,6 +16,9 @@ env:
 jobs:
   build-base-container:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
@@ -24,11 +27,11 @@ jobs:
       with:
         registry: ${{ env.REGISTRY }}
         username: ${{ github.repository_owner }}
-        password: ${{ secrets.REGISTRY_TOKEN }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     - name: Setup Docker Buildx
       uses: docker/setup-buildx-action@v3
     - name: Build and Push Docker Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         push: true     # Will only build if this is not here
         tags: ${{ env.REGISTRY }}/${{ github.repository }}:base

.github/workflows/build_latest_container.yml

@@ -20,6 +20,9 @@ env:
 jobs:
   build-latest-container:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - name: Checkout Code
       uses: actions/checkout@v4
@@ -28,9 +31,9 @@ jobs:
       with:
         registry: ${{ env.REGISTRY }}
         username: ${{ github.repository_owner }}
-        password: ${{ secrets.REGISTRY_TOKEN }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build and Push Docker Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         file: ./Dockerfile.prod
         push: true       # Will only build if this is not here

.github/workflows/build_tagged_container.yml

@@ -10,6 +10,9 @@ env:
 jobs:
   build-tagged-container:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
     steps:
     - name: Get tag
       id: tag
@@ -24,9 +27,9 @@ jobs:
       with:
         registry: ${{ env.REGISTRY }}
         username: ${{ github.repository_owner }}
-        password: ${{ secrets.REGISTRY_TOKEN }}
+        password: ${{ secrets.GITHUB_TOKEN }}
     - name: Build and Push Docker Image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         file: ./Dockerfile.prod
         push: true             # Will only build if this is not here

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
 - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-  rev: v2.9.0
+  rev: v2.14.0
   hooks:
   - id: pretty-format-yaml
     args: [--autofix, --indent, '2']
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v5.0.0
   hooks:
   - id: check-yaml
   - id: end-of-file-fixer
@@ -33,7 +33,7 @@ repos:
   - id: beautysh
 
 - repo: https://github.com/igorshubovych/markdownlint-cli
-  rev: v0.34.0
+  rev: v0.42.0
   hooks:
   - id: markdownlint-fix
     entry: markdownlint -f README.md --disable MD013 MD036 MD033 MD046

CMakeLists.txt

@@ -70,6 +70,17 @@ if(Boost_FOUND)
     set(TEMP "${TEMP} ") # A space to cleanly separate from existing content
   endif()
 
+  # Find CUDA architecture
+  include(FindCUDA/select_compute_arch)
+  cuda_detect_installed_gpus(INSTALLED_GPU_CCS_1)
+  string(STRIP "${INSTALLED_GPU_CCS_1}" INSTALLED_GPU_CCS_2)
+  string(REPLACE " " ";" INSTALLED_GPU_CCS_3 "${INSTALLED_GPU_CCS_2}")
+  string(REPLACE "." "" INSTALLED_GPU_CCS_4 "${INSTALLED_GPU_CCS_3}")
+  string(REPLACE "+PTX" "" CUDA_ARCH_LIST "${INSTALLED_GPU_CCS_4}")
+  set(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH_LIST})
+  set_target_properties(gpuvmem PROPERTIES CUDA_ARCHITECTURES
+                                           "${CUDA_ARCH_LIST}")
+
   message(STATUS "CUDA Architecture: ${CMAKE_CUDA_ARCHITECTURES}")
   message(STATUS "CUDA Version: ${CUDAToolkit_VERSION}")
   message(STATUS "CUDA Path: ${CUDAToolkit_LIBRARY_ROOT}")

Dockerfile

@@ -43,7 +43,7 @@ RUN git clone --single-branch --branch v3.5.0 https://github.com/casacore/casaco
 
 # Install CUDA samples
 RUN cd /usr/local/cuda && \
-    git clone --single-branch --branch v11.6 https://github.com/NVIDIA/cuda-samples.git samples && \
+    git clone --single-branch --branch v12.4 https://github.com/NVIDIA/cuda-samples.git samples && \
     cd samples && \
     mv Common common && \
     mv Samples samples && \

Dockerfile.prod

@@ -14,6 +14,7 @@ RUN cd gpuvmem && \
     cd $BUILD_DIR && \
     cmake --build . --target install --verbose -j `nproc` && \
     cd .. && \
-    pip3 install -r requirements.txt
+    pip3 install -r requirements.txt && \
+    mkdir -p $HOME/.casa/data
 RUN echo "Hello there! from GPUVMEM production image"
 LABEL org.opencontainers.image.source="https://github.com/miguelcarcamov/gpuvmem"

environment.yml

@@ -12,15 +12,14 @@ dependencies:
 - numpy=1.24
 - pip
 - pip:
-  - casadata==2023.9.18
-  - casafeather==0.0.20
-  - casalogger==1.0.17
-  - casampi==0.5.4
-  - casaplotms==2.2.4
-  - casaplotserver==1.7.1
-  - casashell==6.6.0.20
-  - casatablebrowser==0.0.33
-  - casatasks==6.6.0.20
-  - casatestutils==6.6.0.20
-  - casatools==6.6.0.20
-  - casaviewer==1.9.1
+  - casatools==6.6.4.34
+  - casatasks==6.6.4.34
+  - casaplotms==2.4.3
+  - casaviewer==2.1.4
+  - casashell==6.6.4.34
+  - casaplotserver==1.8.2
+  - casatestutils==6.6.4.34
+  - casatablebrowser==0.0.35
+  - casalogger==1.0.19
+  - casafeather==0.0.22
+  - casampi==0.5.5

environment_cudatoolkit.yml

@@ -15,15 +15,14 @@ dependencies:
 - numpy=1.24
 - pip
 - pip:
-  - casadata==2023.9.18
-  - casafeather==0.0.20
-  - casalogger==1.0.17
-  - casampi==0.5.4
-  - casaplotms==2.2.4
-  - casaplotserver==1.7.1
-  - casashell==6.6.0.20
-  - casatablebrowser==0.0.33
-  - casatasks==6.6.0.20
-  - casatestutils==6.6.0.20
-  - casatools==6.6.0.20
-  - casaviewer==1.9.1
+  - casatools==6.6.4.34
+  - casatasks==6.6.4.34
+  - casaplotms==2.4.3
+  - casaviewer==2.1.4
+  - casashell==6.6.4.34
+  - casaplotserver==1.8.2
+  - casatestutils==6.6.4.34
+  - casatablebrowser==0.0.35
+  - casalogger==1.0.19
+  - casafeather==0.0.22
+  - casampi==0.5.5

include/MSFITSIO.cuh

@@ -104,8 +104,10 @@ typedef struct field {
   int valid_frequencies;
   double ref_ra, ref_dec;
   double phs_ra, phs_dec;
-  float ref_xobs, ref_yobs;
-  float phs_xobs, phs_yobs;
+  float ref_xobs_cartesian, ref_yobs_cartesian;
+  float phs_xobs_cartesian, phs_yobs_cartesian;
+  float ref_xobs_pix, ref_yobs_pix;
+  float phs_xobs_pix, phs_yobs_pix;
   float* atten_image;
   std::vector<float> nu;
   std::vector<std::vector<long>> numVisibilitiesPerFreqPerStoke;

include/chi2.cuh

@@ -11,13 +11,19 @@ class Chi2 : public Fi {
   void calcGi(float* p, float* xi);
   void restartDGi();
   void addToDphi(float* device_dphi);
-  void configure(int penalizatorIndex, int imageIndex, int imageToAdd) override;
-  void setPenalizationFactorFromInputIndex(int index){};
-  float calculateSecondDerivate(){};
+  void configure(int penalizatorIndex,
+                 int imageIndex,
+                 int imageToAdd,
+                 bool normalize) override;
+  void setPenalizationFactorFromInputIndex(int index) {};
+  float calculateSecondDerivate() {};
   void setCKernel(CKernel* ckernel) override;
+  void setFgScale(float fg_scale) override;
+  float getFgScale() override;
 
  private:
   VirtualImageProcessor* ip;
+  float fg_scale = 1.0;
   int imageToAdd;
   float* result_dchi2;
 };

include/classes/fi.cuh

@@ -17,23 +17,27 @@ class Fi {
     this->penalization_factor = 1.0f;
     this->Inu = NULL;
     this->iteration = 0;
+    this->normalize = false;
   }
 
   virtual float calcFi(float* p) = 0;
   virtual void calcGi(float* p, float* xi) = 0;
   virtual void restartDGi() = 0;
   virtual void addToDphi(float* device_dphi) = 0;
-  virtual void setPrior(float prior){};
-  virtual void setPrior(float* prior){};
-  virtual float getEta(){};
-  virtual void setEta(float eta){};
-  virtual void setCKernel(CKernel* ckernel){};
+  virtual void setPrior(float prior) {};
+  virtual void setPrior(float* prior) {};
+  virtual float getEta() {};
+  virtual void setEta(float eta) {};
+  virtual void setCKernel(CKernel* ckernel) {};
+  virtual void setFgScale(float fg_scale) {};
+  virtual float getFgScale() {};
 
   std::string getName() { return this->name; };
 
   std::string setName(std::string name) { this->name = name; };
 
   float get_fivalue() { return this->fi_value; };
+  bool getNormalize() { return this->normalize; };
   float getPenalizationFactor() { return this->penalization_factor; };
   void set_fivalue(float fi) { this->fi_value = fi; };
   void setPenalizationFactor(float p) { this->penalization_factor = p; };
@@ -48,13 +52,18 @@ class Fi {
     this->device_DS = DS;
   };
   void setIteration(int iteration) { this->iteration = iteration; };
+  void setNormalize(bool normalize) { this->normalize = normalize; };
 
   virtual float calculateSecondDerivate() = 0;
-  virtual void configure(int penalizatorIndex, int imageIndex, int imageToAdd) {
+  virtual void configure(int penalizatorIndex,
+                         int imageIndex,
+                         int imageToAdd,
+                         bool normalize) {
     this->imageIndex = imageIndex;
     this->order = order;
     this->mod = mod;
     this->imageToAdd = imageToAdd;
+    this->normalize = normalize;
 
     if (imageIndex > image_count - 1 || imageToAdd > image_count - 1) {
       printf("There is no image for the provided index %s\n", this->name);
@@ -89,9 +98,9 @@ class Fi {
   int mod;
   int order;
   std::string name;
-  ;
   cufftComplex* Inu;
   int imageToAdd;
+  bool normalize;
 };
 
 #endif

include/classes/synthesizer.cuh

@@ -70,6 +70,10 @@ class Synthesizer {
 
   __host__ void setVisNoise(float noise) { this->vis_noise = noise; };
 
+  __host__ float getFgScale() { return this->fg_scale; };
+
+  __host__ void setFgScale(float fg_scale) { this->fg_scale = fg_scale; };
+
  protected:
   cufftComplex* device_I;
   Image* image;
@@ -81,6 +85,7 @@ class Synthesizer {
   Error* error = NULL;
   int griddingThreads = 0;
   bool gridding = false;
+  float fg_scale = 1.0;
   float vis_noise = 0.0;
   void (*Order)(Optimizer* o, Image* I) = NULL;
   int imagesChanged = 0;

include/classes/virtualimageprocessor.cuh

@@ -15,9 +15,10 @@ class VirtualImageProcessor {
                           float xobs,
                           float yobs,
                           float freq,
-                          int primary_beam) = 0;
+                          int primary_beam,
+                          float fg_scale) = 0;
   virtual void calculateInu(cufftComplex* image, float* I, float freq) = 0;
-  virtual void chainRule(float* I, float freq) = 0;
+  virtual void chainRule(float* I, float freq, float fg_scale) = 0;
   virtual void configure(int i) = 0;
   virtual CKernel* getCKernel() { return this->ckernel; };
   virtual void setCKernel(CKernel* ckernel) { this->ckernel = ckernel; };

include/functions.cuh

@@ -145,13 +145,18 @@ __host__ T deviceReduce(T* in, long N, int input_threads);
 __host__ float deviceMaxReduce(float* in, long N, int input_threads);
 __host__ float deviceMinReduce(float* in, long N, int input_threads);
 __host__ float simulate(float* I, VirtualImageProcessor* ip);
-__host__ float chi2(float* I, VirtualImageProcessor* ip);
+__host__ float chi2(float* I,
+                    VirtualImageProcessor* ip,
+                    bool normalize,
+                    float fg_scale);
 __host__ void linkRestartDGi(float* dgi);
 __host__ void linkAddToDPhi(float* dphi, float* dgi, int index);
 __host__ void dchi2(float* I,
                     float* dxi2,
                     float* result_dchi2,
-                    VirtualImageProcessor* ip);
+                    VirtualImageProcessor* ip,
+                    bool normalize,
+                    float fg_scale);
 __host__ void defaultNewP(float* p, float* xi, float xmin, int image);
 __host__ void particularNewP(float* p, float* xi, float xmin, int image);
 __host__ void defaultEvaluateXt(float* xt,
@@ -171,10 +176,11 @@ __host__ void linkApplyBeam2I(cufftComplex* image,
                               float xobs,
                               float yobs,
                               float freq,
-                              int primary_beam);
+                              int primary_beam,
+                              float fg_scale);
 __host__ void linkClipWNoise2I(float* I);
 __host__ void linkCalculateInu2I(cufftComplex* image, float* I, float freq);
-__host__ void linkChain2I(float* chain, float freq, float* I);
+__host__ void linkChain2I(float* chain, float freq, float* I, float fg_scale);
 __host__ void normalizeImage(float* image, float normalization_factor);
 __host__ void linkClip(float* I);
 __host__ void DEntropy(float* I,
@@ -485,7 +491,8 @@ __global__ void DChi2(float* noise,
                       float pb_factor,
                       float pb_cutoff,
                       float freq,
-                      int primary_beam);
+                      int primary_beam,
+                      bool normalize);
 __global__ void DChi2(float* noise,
                       float* gcf,
                       float* dChi2,
@@ -506,7 +513,8 @@ __global__ void DChi2(float* noise,
                       float pb_factor,
                       float pb_cutoff,
                       float freq,
-                      int primary_beam);
+                      int primary_beam,
+                      bool normalize);
 __global__ void projection(float* px, float* x, float MINPIX, long N);
 __global__ void substraction(float* x,
                              cufftComplex* xc,

include/imageProcessor.cuh

@@ -15,9 +15,10 @@ class ImageProcessor : public VirtualImageProcessor {
                   float xobs,
                   float yobs,
                   float freq,
-                  int primary_beam);
+                  int primary_beam,
+                  float fg_scale);
   void calculateInu(cufftComplex* image, float* I, float freq);
-  void chainRule(float* I, float freq);
+  void chainRule(float* I, float freq, float fg_scale);
   void configure(int i);
 };