Skip to content

Commit

Permalink
make gradle build and add github actions
Browse files Browse the repository at this point in the history
  • Loading branch information
lfoppiano committed Mar 29, 2024
1 parent e887a61 commit 5e8ecfd
Show file tree
Hide file tree
Showing 7 changed files with 263 additions and 85 deletions.
42 changes: 42 additions & 0 deletions .github/workflows/ci-build-manual.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Build and push a development version on docker

on:
workflow_dispatch:


jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17.0.10+7'
distribution: 'temurin'
cache: 'gradle'
- name: Build with Gradle
run: ./gradlew build -x test

docker-build:
needs: [ build ]
runs-on: ubuntu-latest

steps:
- name: Create more disk space
run: sudo rm -rf /usr/share/dotnet && sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" && sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- uses: actions/checkout@v2
- name: Build and push
id: docker_build
uses: mr-smithers-excellent/docker-build-push@v6
with:
dockerfile: Dockerfile.local
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
image: lfoppiano/datastet
registry: docker.io
pushImage: true
tags: latest-develop
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
65 changes: 65 additions & 0 deletions .github/workflows/ci-build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Build unstable

on: [push]

concurrency:
group: gradle
# cancel-in-progress: true


jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17.0.10+7'
distribution: 'temurin'
cache: 'gradle'
- name: Build with Gradle
run: ./gradlew build -x test

- name: Test with Gradle Jacoco and Coveralls
run: ./gradlew test jacocoTestReport coveralls --no-daemon

- name: Coveralls GitHub Action
uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
format: jacoco

docker-build:
needs: [ build ]
runs-on: ubuntu-latest

steps:
# - name: Maximize build disk space
# uses: easimon/maximize-build-space@v10
# with:
# root-reserve-mb: 512
# swap-size-mb: 1024
# remove-dotnet: 'true'
- name: Create more disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
sudo rm -rf /opt/hostedtoolcache
- uses: actions/checkout@v4
- name: Build and push
id: docker_build
uses: mr-smithers-excellent/docker-build-push@v6
with:
dockerfile: Dockerfile.local
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
image: lfoppiano/datastet
registry: docker.io
pushImage: ${{ github.event_name != 'pull_request' }}
tags: latest-develop
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
32 changes: 32 additions & 0 deletions .github/workflows/ci-integration-manual.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Run integration tests manually

on:
push:
branches:
- master
workflow_dispatch:

jobs:
build:
runs-on: ubuntu-latest

steps:
- name: Checkout grobid home
uses: actions/checkout@v4
with:
repository: kermitt2/grobid
path: ./grobid
- name: Checkout Datastet
uses: actions/checkout@v4
with:
path: ./grobid/datastet
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17.0.10+7'
distribution: 'temurin'
cache: 'gradle'
- name: Build and run integration tests
working-directory: ./grobid/datastet
run: ./gradlew copyModels integration --no-daemon

74 changes: 74 additions & 0 deletions .github/workflows/ci-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
name: Build release

on:
workflow_dispatch:
push:
tags:
- 'v*'

concurrency:
group: docker
cancel-in-progress: true


jobs:
build:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17.0.10+7'
distribution: 'temurin'
cache: 'gradle'
- name: Build with Gradle
run: ./gradlew build -x test

- name: Test with Gradle Jacoco and Coveralls
run: ./gradlew test jacocoTestReport coveralls --no-daemon

- name: Coveralls GitHub Action
uses: coverallsapp/github-action@v2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
format: jacoco


docker-build:
needs: [build]
runs-on: ubuntu-latest

steps:
- name: Create more disk space
run: sudo rm -rf /usr/share/dotnet && sudo rm -rf /opt/ghc && sudo rm -rf "/usr/local/share/boost" && sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Set tags
id: set_tags
run: |
DOCKER_IMAGE=lfoppiano/datastet
VERSION=""
if [[ $GITHUB_REF == refs/tags/v* ]]; then
VERSION=${GITHUB_REF#refs/tags/v}
fi
if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
TAGS="${VERSION}"
else
TAGS="latest"
fi
echo "TAGS=${TAGS}"
echo ::set-output name=tags::${TAGS}
- uses: actions/checkout@v4
- name: Build and push
id: docker_build
uses: mr-smithers-excellent/docker-build-push@v6
with:
dockerfile: Dockerfile.local
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
image: lfoppiano/datastet
registry: docker.io
pushImage: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.set_tags.outputs.tags }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
31 changes: 11 additions & 20 deletions src/test/java/org/grobid/core/engines/DataseerClassifierTest.java
Original file line number Diff line number Diff line change
@@ -1,49 +1,40 @@
package org.grobid.core.engines;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.apache.commons.io.IOUtils;
import org.grobid.core.document.Document;
import org.grobid.core.factory.GrobidFactory;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.utilities.DataseerConfiguration;
import org.grobid.core.main.GrobidHomeFinder;
import org.grobid.core.utilities.GrobidConfig.ModelParameters;
import org.grobid.core.main.LibraryLoader;
import org.grobid.core.utilities.DatastetConfiguration;
import org.grobid.core.utilities.GrobidConfig.ModelParameters;
import org.grobid.core.utilities.GrobidProperties;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Ignore;
import org.junit.Test;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Arrays;
import java.util.ArrayList;

import org.apache.commons.lang3.tuple.Pair;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertNotNull;
import java.util.Arrays;
import java.util.List;

/**
* @author Patrice
*/
@Ignore
public class DataseerClassifierTest {
private static DataseerConfiguration configuration;
private static DatastetConfiguration configuration;

@BeforeClass
public static void setUpClass() throws Exception {
DataseerConfiguration dataseerConfiguration = null;
DatastetConfiguration dataseerConfiguration = null;
try {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());

File yamlFile = new File("resources/config/dataseer-ml.yml").getAbsoluteFile();
yamlFile = new File(yamlFile.getAbsolutePath());
dataseerConfiguration = mapper.readValue(yamlFile, DataseerConfiguration.class);
dataseerConfiguration = mapper.readValue(yamlFile, DatastetConfiguration.class);

String pGrobidHome = dataseerConfiguration.getGrobidHome();

Expand Down
32 changes: 12 additions & 20 deletions src/test/java/org/grobid/core/engines/DatasetParserTest.java
Original file line number Diff line number Diff line change
@@ -1,50 +1,42 @@
package org.grobid.core.engines;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import org.apache.commons.io.IOUtils;
import org.grobid.core.document.Document;
import org.grobid.core.data.Dataset;
import org.grobid.core.factory.GrobidFactory;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.core.utilities.DataseerConfiguration;
import org.grobid.core.main.GrobidHomeFinder;
import org.grobid.core.utilities.GrobidConfig.ModelParameters;
import org.grobid.core.main.LibraryLoader;
import org.grobid.core.utilities.DatastetConfiguration;
import org.grobid.core.utilities.GrobidConfig.ModelParameters;
import org.grobid.core.utilities.GrobidProperties;
import org.grobid.service.configuration.DatastetServiceConfiguration;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.Ignore;
import org.junit.Test;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Arrays;
import java.util.ArrayList;

import org.apache.commons.lang3.tuple.Pair;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.hasSize;
import static org.junit.Assert.assertNotNull;
import java.util.Arrays;
import java.util.List;

/**
* @author Patrice
*/
@Ignore
public class DatasetParserTest {
private static DataseerConfiguration configuration;
private static DatastetConfiguration configuration;

@BeforeClass
public static void setUpClass() throws Exception {
DataseerConfiguration dataseerConfiguration = null;
DatastetConfiguration dataseerConfiguration = null;
try {
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());

File yamlFile = new File("resources/config/dataseer-ml.yml").getAbsoluteFile();
yamlFile = new File(yamlFile.getAbsolutePath());
dataseerConfiguration = mapper.readValue(yamlFile, DataseerConfiguration.class);
dataseerConfiguration = mapper.readValue(yamlFile, DatastetConfiguration.class);

String pGrobidHome = dataseerConfiguration.getGrobidHome();

Expand Down
Loading

0 comments on commit 5e8ecfd

Please sign in to comment.