diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 6e64999..0000000 --- a/.coveralls.yml +++ /dev/null @@ -1 +0,0 @@ -service_name: travis-ci \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a707323 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" # Location of package manifests + schedule: + interval: "monthly" diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..1ecd864 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,28 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: Build with Maven + run: mvn -B package --file pom.xml + - name: Run test and coverage report + run: mvn clean test jacoco:report diff --git a/.gitignore b/.gitignore index cff24b4..d54827e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,9 @@ hs_err_pid* # IntelliJ files .idea/ -*.iml \ No newline at end of file +*.iml + +# Eclipse files +/.classpath +/.project +/.settings/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dcd69a6..0000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -dist: - trusty - -sudo: - false - -language: - java - -jdk: - - oraclejdk8 - -install: mvn install -DskipTests -Dgpg.skip -Dmaven.javadoc.skip=true -B -V - -after_success: - - mvn clean test jacoco:report coveralls:report diff --git a/README.md b/README.md index f602e71..8740992 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,28 @@ - # datapackage-java -A Java library for working with Data Packages. Snapshots on [Jitpack](https://jitpack.io/#frictionlessdata/datapackage-java) - -[![Build Status](https://travis-ci.org/frictionlessdata/datapackage-java.svg?branch=master)](https://travis-ci.org/frictionlessdata/datapackage-java) -[![Coverage Status](https://coveralls.io/repos/github/frictionlessdata/datapackage-java/badge.svg?branch=master)](https://coveralls.io/github/frictionlessdata/datapackage-java?branch=master) [![License](https://img.shields.io/github/license/frictionlessdata/datapackage-java.svg)](https://github.com/frictionlessdata/datapackage-java/blob/master/LICENSE) -[![Github](https://img.shields.io/badge/github-master-brightgreen)](https://github.com/frictionlessdata/datapackage-java/tree/master/) -[![](https://jitpack.io/v/frictionlessdata/datapackage-java.svg)](https://jitpack.io/#frictionlessdata/datapackage-java) -[![Gitter](https://img.shields.io/gitter/room/frictionlessdata/chat.svg)](https://gitter.im/frictionlessdata/chat) +[![Release](https://img.shields.io/jitpack/v/github/frictionlessdata/datapackage-java)](https://jitpack.io/#frictionlessdata/datapackage-java) +[![Codebase](https://img.shields.io/badge/codebase-github-brightgreen)](https://github.com/frictionlessdata/datapackage-java) +[![Support](https://img.shields.io/badge/support-discord-brightgreen)](https://discordapp.com/invite/Sewv6av) + +A Java library for working with Data Packages according to the +[Frictionless Data](https://specs.frictionlessdata.io/data-package/) specifications. +A Data Package is a simple container format for creating self-contained packages of data. It provides the basis +for convenient delivery, installation and management of datasets. It shares some similarity with simple database +formats, but lacks a robust query engine, instead focusing on exchanging bundles of related data. + +Please find releases on [Jitpack](https://jitpack.io/#frictionlessdata/datapackage-java) ## Usage +- [Create a Data Package](#create_a_data_package) explains how to create a Data Package +- [Iterate through Data](#iterate_through_data) explains how to iterate through data in Resources +- [Edit a Data Package](#edit_a_data_package) explains how to add or remove Resources or properties to or from a Data Package +- [Save to File](#save_to_file) explains how to save a Data Package to a file +- [Working with Foreign Keys](docs/foreign-keys.md) explains how to set foreign key constraints in Data Packages +- [Contributing](#contributing) contributions are welcome + ### Create a Data Package #### From JSONObject Object @@ -204,6 +214,7 @@ dp.save("/destination/path/datapackage.zip") Found a problem and would like to fix it? Have that great idea and would love to see it in the repository? +> [!NOTE] > Please open an issue before you start working. It could save a lot of time for everyone and we are super happy to answer questions and help you along the way. Furthermore, feel free to join [frictionlessdata Gitter chat room](https://gitter.im/frictionlessdata/chat) and ask questions. @@ -213,9 +224,9 @@ This project follows the [Open Knowledge International coding standards](https:/ Get started: ```sh # install jabba and maven2 -$ cd tableschema-java -$ jabba install 1.8 -$ jabba use 1.8 +$ cd datapackage-java +$ jabba install 17 +$ jabba use 17 $ mvn install -DskipTests=true -Dmaven.javadoc.skip=true -B -V $ mvn test -B ``` diff --git a/docs/foreign-keys.md b/docs/foreign-keys.md new file mode 100644 index 0000000..104aa2c --- /dev/null +++ b/docs/foreign-keys.md @@ -0,0 +1,80 @@ +# Working with Foreign Keys + +The library supports foreign keys described in the [Table Schema](http://specs.frictionlessdata.io/table-schema/#foreign-keys) specification. It means if your data package descriptor use `resources[].schema.foreignKeys` property for some resources a data integrity will be checked on reading operations. + +Consider we have a data package: + +```json +{ + "name": "foreign-keys", + "resources": [ + { + "name": "teams", + "data": [ + ["id", "name", "city"], + ["1", "Arsenal", "London"], + ["2", "Real", "Madrid"], + ["3", "Bayern", "Munich"] + ], + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ], + "foreignKeys": [ + { + "fields": "city", + "reference": { + "resource": "cities", + "fields": "name" + } + } + ] + } + }, + { + "name": "cities", + "data": [ + ["name", "country"], + ["London", "England"], + ["Madrid", "Spain"] + ] + } + ] +} +``` + +Let's check relations for a `teams` resource: + +````java +Package dp = new Package(DESCRIPTOR, Paths.get(""), true); + Resource teams = dp.getResource("teams"); + DataPackageValidationException dpe + = Assertions.assertThrows(DataPackageValidationException.class, () -> teams.checkRelations(dp)); + Assertions.assertEquals("Error reading data with relations: Foreign key validation failed: [city] -> [name]: 'Munich' not found in resource 'cities'.", dpe.getMessage()); +```` +As we could see, we can read the Datapackage, but if we call `teams.checkRelations(dp)`, there is a foreign key violation. That's because our lookup table `cities` doesn't have a city of `Munich` but we have a team from there. We need to fix it in `cities` resource: + +````json +{ + "name": "cities", + "data": [ + ["name", "country"], + ["London", "England"], + ["Madrid", "Spain"], + ["Munich", "Germany"] + ] + } +```` + +Now, calling `teams.checkRelations(dp)` will no longer throw an exception. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 6d0da70..fd724f1 100644 --- a/pom.xml +++ b/pom.xml @@ -1,9 +1,10 @@ - + 4.0.0 io.frictionlessdata datapackage-java - 1.0-SNAPSHOT + 0.10.0-SNAPSHOT jar https://github.com/frictionlessdata/datapackage-java/issues @@ -17,27 +18,30 @@ UTF-8 - 1.8 - 1.8 - 80a610e6b2 - 23.6-jre - 1.3 - 5.4.2 - 3.9 - 1.7 - 1.5.1 - 2.9.9 - 3.8.1 - 3.0.1 - 3.0.1 - 3.1.0 - 2.22.0 - 2.8.2 - 1.6 - 2.5.3 - 1.6.8 + UTF-8 + 17 + ${java.version} + ${java.version} + ${java.version} + 0.10.0 + 6.0.1 + 2.0.17 + 4.5.0 + 3.14.1 + 3.9.0 + 3.3.1 + 3.12.0 + 3.3.1 + 3.5.4 + 3.1.4 + 3.0.1 + 3.1.1 + 1.7.0 4.3.0 - 0.8.5 + 12.1.8 + 0.8.14 + 4.9.8.1 + 1.13.0 @@ -56,6 +60,7 @@ utf-8 ${maven.compiler.source} ${maven.compiler.target} + ${maven.compiler.compiler} @@ -72,8 +77,7 @@ org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-plugin.version} - - + @@ -124,67 +128,79 @@ - org.apache.maven.plugins - maven-gpg-plugin - ${maven-gpg-plugin.version} + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} - sign-artifacts - verify + prepare-agent + + prepare-agent + + + + report + prepare-package - sign + report - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-maven-plugin.version} - true - - ossrh - https://oss.sonatype.org/ - true - + org.owasp + dependency-check-maven + ${dependency-check-maven.version} + + + package + + check + + + org.apache.maven.plugins - maven-release-plugin - ${maven-release-plugin.version} - - true - false - forked-path - -Dgpg.passphrase=${gpg.passphrase} - + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + analyze-report + package + + analyze-report + + + ${project.build.directory}/dependency-report + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs-maven-plugin.version} - org.apache.maven.scm - maven-scm-provider-gitexe - 1.9.4 + com.h3xstream.findsecbugs + findsecbugs-plugin + ${findsecbugs-plugin.version} - - - - - org.eluder.coveralls - coveralls-maven-plugin - ${coveralls-maven-plugin.version} - - - - org.jacoco - jacoco-maven-plugin - ${jacoco-maven-plugin.version} + + Max + High + false + - prepare-agent + package - prepare-agent + check @@ -200,73 +216,36 @@ - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - org.junit.vintage - junit-vintage-engine - ${junit.version} - test - - - - - org.hamcrest - hamcrest-all - ${hamcrest.version} - test - + + + org.slf4j + slf4j-simple + ${slf4j-simple.version} + test + - + org.apache.commons - commons-lang3 - ${apache-commons-lang3.version} - - - - - org.everit.json - org.everit.json.schema - ${everit-json-schema.version} - - - - - joda-time - joda-time - ${joda-time.version} - - - - - - org.apache.commons - commons-csv - ${apache-commons-csv.version} - - - - - com.google.guava - guava - ${google-guava.version} + commons-collections4 + ${apache-commons-collections4.version} - com.github.frictionlessdata tableschema-java ${tableschema-java-version} + + + org.junit.jupiter + junit-jupiter-api + ${junit.version} + test + - \ No newline at end of file + diff --git a/src/main/java/io/frictionlessdata/datapackage/BaseInterface.java b/src/main/java/io/frictionlessdata/datapackage/BaseInterface.java new file mode 100644 index 0000000..710e9c0 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/BaseInterface.java @@ -0,0 +1,108 @@ +package io.frictionlessdata.datapackage; + +import java.util.List; + +public interface BaseInterface { + + /** + * @return the name + */ + String getName(); + + /** + * @param name the name to set + */ + void setName(String name); + + /** + * @return the profile + */ + String getProfile(); + + /** + * @param profile the profile to set + */ + void setProfile(String profile); + + /** + * @return the title + */ + String getTitle(); + + /** + * @param title the title to set + */ + void setTitle(String title); + + /** + * @return the description + */ + String getDescription(); + + /** + * @param description the description to set + */ + void setDescription(String description); + + + /** + * @return the sources + */ + List getSources(); + + /** + * @param sources the sources to set + */ + void setSources(List sources); + + /** + * @return the licenses + */ + List getLicenses(); + + /** + * @param licenses the licenses to set + */ + void setLicenses(List licenses); + + /** + * @return the encoding + */ + String getEncoding(); + + /** + * @param encoding the encoding to set + */ + void setEncoding(String encoding); + + /** + * @return the bytes + */ + Integer getBytes(); + + /** + * @param bytes the bytes to set + */ + void setBytes(Integer bytes); + + /** + * @return the hash + */ + String getHash(); + + /** + * @param hash the hash to set + */ + void setHash(String hash); + + /** + * @return the mediaType + */ + String getMediaType(); + + /** + * @param mediaType the mediaType to set + */ + void setMediaType(String mediaType); + +} diff --git a/src/main/java/io/frictionlessdata/datapackage/Contributor.java b/src/main/java/io/frictionlessdata/datapackage/Contributor.java index 1821bb1..83053b5 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Contributor.java +++ b/src/main/java/io/frictionlessdata/datapackage/Contributor.java @@ -1,76 +1,77 @@ package io.frictionlessdata.datapackage; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.exc.InvalidFormatException; +import tools.jackson.databind.node.ArrayNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import org.json.JSONArray; -import org.json.JSONObject; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.lang3.StringUtils; import java.net.MalformedURLException; import java.net.URL; -import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; -import static io.frictionlessdata.datapackage.Package.isValidUrl; - +@JsonPropertyOrder({ + "title", + "email", + "path", + "role", + "organization" +}) public class Contributor { + static final String invalidUrlMsg = "URLs for contributors must be fully qualified"; private String title; private String email; private URL path; - private Role role; + private String role; private String organization; - /** - * Create a new Contributor object from a JSON representation - * @param jsonObj JSON representation, eg. from Package definition - * @return new Dialect object with values from JSONObject - */ - public static Contributor fromJson(JSONObject jsonObj) throws MalformedURLException { - if (null == jsonObj) - return null; - Contributor c = new Contributor(); - if (jsonObj.has("title")) - c.title = jsonObj.getString("title"); - if (jsonObj.has("email")) - c.title = jsonObj.getString("email"); - if (jsonObj.has("path")) { - URL url = new URL(jsonObj.getString("path")); - if (isValidUrl(url)) { - c.path = url; - } else { - throw new DataPackageException("URLs for contributors must be fully qualified"); - } - } - if (jsonObj.has("role")) { - String role = jsonObj.getString("role"); - c.role = Role.valueOf(role.toUpperCase()); - } - if (jsonObj.has("organization")) - c.title = jsonObj.getString("organization"); - return c; - } + public String getTitle() { + return title; + } - /** - * Create a new Contributor object from a JSON representation - * @param jsonArr JSON representation, eg. from Package definition - * @return new Dialect object with values from JSONObject - */ - public static Collection fromJson(JSONArray jsonArr) throws MalformedURLException { - final Collection contributors = new ArrayList<>(); - for (int cnt = 0; cnt < jsonArr.length(); cnt++) { - JSONObject obj = jsonArr.getJSONObject(cnt); - contributors.add(fromJson(obj)); - }; - return contributors; - } + public String getEmail() { + return email; + } - public static Collection fromJson(String json) throws MalformedURLException { - return fromJson(new JSONArray(json)); - } + public URL getPath() { + return path; + } - public static enum Role { - AUTHOR, - PUBLISHER, - MAINTAINER, - WRANGLER, - CONTRIBUTOR + public String getRole() { + return role; + } + + public String getOrganization() { + return organization; + } + + public static Collection fromJson(JsonNode json) { + if ((null == json) || json.isEmpty() || (!(json instanceof ArrayNode))) { + return null; + } + + try { + return JsonUtil.getInstance().deserialize(json, new TypeReference<>() {}); + } catch (Exception ex) { + Throwable cause = ex.getCause(); + if (ex.getClass().isAssignableFrom(InvalidFormatException.class)) { + if (Objects.nonNull(cause) && cause.getClass().isAssignableFrom(MalformedURLException.class)) { + throw new DataPackageException(invalidUrlMsg); + } + } + throw new DataPackageException(ex); + } } + + public static Collection fromJson(String json) { + if (StringUtils.isEmpty(json)) { + return null; + } + JsonNode jsonNode = JsonUtil.getInstance().readValue(json); + return fromJson(jsonNode); + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/Dialect.java b/src/main/java/io/frictionlessdata/datapackage/Dialect.java index ee5b498..a553553 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Dialect.java +++ b/src/main/java/io/frictionlessdata/datapackage/Dialect.java @@ -1,10 +1,20 @@ package io.frictionlessdata.datapackage; +import com.fasterxml.jackson.annotation.*; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import tools.jackson.databind.JsonNode; +import io.frictionlessdata.tableschema.io.FileReference; +import io.frictionlessdata.tableschema.util.JsonUtil; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.QuoteMode; -import org.json.JSONObject; import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; /** * CSV Dialect defines a simple format to describe the various dialects of CSV files in a language agnostic @@ -18,25 +28,41 @@ * According to specification: https://frictionlessdata.io/specs/csv-dialect/ */ -public class Dialect { +@JsonInclude(value = Include.NON_NULL, content = Include.NON_EMPTY) +@JsonPropertyOrder({ + "caseSensitiveHeader", + "quoteChar", + "doubleQuote", + "delimiter", + "lineTerminator", + "nullSequence", + "header", + "csvddfVersion", + "skipInitialSpace" +}) +@JsonIgnoreProperties(ignoreUnknown = true) +public class Dialect implements Cloneable { + + private FileReference reference; + // we construct one instance that will always keep the default values - public static Dialect DEFAULT = new Dialect(){ - private JSONObject jsonObject; + public static final Dialect DEFAULT = new Dialect(){ + private JsonNode JsonNode; - public String getJson() { + public String asJson() { lazyCreate(); - return jsonObject.toString(); + return JsonNode.toString(); } Object get(String key) { lazyCreate(); - return jsonObject.get(key); + return JsonNode.get(key); } private void lazyCreate() { - if (null == jsonObject) { + if (null == JsonNode) { Dialect newDialect = new Dialect(); - jsonObject = newDialect.getJsonObject(false); + JsonNode = newDialect.getJsonNode(false); } } }; @@ -70,7 +96,7 @@ private void lazyCreate() { /** * specifies the null sequence (for example \N). Not set by default */ - private String nullSequence = null; + private String nullSequence = ""; /** * specifies how to interpret whitespace which immediately follows a delimiter; @@ -100,8 +126,24 @@ private void lazyCreate() { * a number, in n.n format, e.g., 1.2. If not present, consumers should assume latest schema version. */ private Double csvddfVersion = 1.2; + + /** + * Any extra properties that arent defined explicitly + */ + private Map additionalProperties = new HashMap<>(); + + @JsonIgnore + public FileReference getReference() { + return reference; + } + + public void setReference (FileReference ref){ + reference = ref; + } - public Dialect clone() { + @Override + public Dialect clone() throws CloneNotSupportedException { + super.clone(); Dialect retVal = new Dialect(); retVal.delimiter = this.delimiter; retVal.escapeChar = this.escapeChar; @@ -119,23 +161,24 @@ public Dialect clone() { // will fail for multi-character delimiters. Oh my... public CSVFormat toCsvFormat() { - CSVFormat format = CSVFormat.DEFAULT - .withDelimiter(delimiter.charAt(0)) - .withEscape(escapeChar) - .withIgnoreSurroundingSpaces(skipInitialSpace) - .withNullString(nullSequence) - .withCommentMarker(commentChar) - .withSkipHeaderRecord(!hasHeaderRow) - .withQuote(quoteChar) - .withQuoteMode(doubleQuote ? QuoteMode.MINIMAL : QuoteMode.NONE); + CSVFormat format = CSVFormat.DEFAULT.builder() + .setDelimiter(delimiter.charAt(0)) + .setEscape(escapeChar) + .setIgnoreSurroundingSpaces(skipInitialSpace) + .setNullString(nullSequence) + .setCommentMarker(commentChar) + .setSkipHeaderRecord(!hasHeaderRow) + .setQuote(quoteChar) + .setQuoteMode(doubleQuote ? QuoteMode.MINIMAL : QuoteMode.NONE) + .get(); if (hasHeaderRow) - format = format.withHeader(); + format = format.builder().setHeader().get(); return format; } public static Dialect fromCsvFormat(CSVFormat format) { Dialect dialect = new Dialect(); - dialect.setDelimiter(format.getDelimiter()+""); + dialect.setDelimiter(format.getDelimiterString()); dialect.setEscapeChar(format.getEscapeCharacter()); dialect.setSkipInitialSpace(format.getIgnoreSurroundingSpaces()); dialect.setNullSequence(format.getNullString()); @@ -148,64 +191,47 @@ public static Dialect fromCsvFormat(CSVFormat format) { return dialect; } + /** + * Read, create, and validate a Dialect from a FileReference. + * + * @param reference the File or URL to read dialect JSON data from + * @throws Exception thrown if reading from the stream or parsing throws an exception + */ + public static Dialect fromJson (FileReference reference) throws Exception { + String dialectString = null; + try (InputStreamReader ir = new InputStreamReader(reference.getInputStream(), StandardCharsets.UTF_8); + BufferedReader br = new BufferedReader(ir)){ + dialectString = br.lines().collect(Collectors.joining("\n")); + } + Dialect dialect = fromJson (dialectString); + dialect.reference = reference; + reference.close(); + return dialect; + } + /** * Create a new Dialect object from a JSON representation * @param json JSON as String representation, eg. from Resource definition - * @return new Dialect object with values from JSONObject + * @return new Dialect object with values from JsonNode */ public static Dialect fromJson(String json) { if (null == json) return null; - JSONObject jsonObj = new JSONObject(json); - Dialect dialect = new Dialect(); - if (jsonObj.has("delimiter")) { - dialect.setDelimiter(jsonObj.getString("delimiter")); - } - if (jsonObj.has("escapeChar")) - dialect.setEscapeChar(jsonObj.getString("escapeChar").charAt(0)); - if (jsonObj.has("skipInitialSpace")) - dialect.setSkipInitialSpace(jsonObj.getBoolean("skipInitialSpace")); - if (jsonObj.has("nullSequence")) - dialect.setNullSequence(jsonObj.getString("nullSequence")); - if (jsonObj.has("commentChar")) - dialect.setCommentChar(jsonObj.getString("commentChar").charAt(0)); - if (jsonObj.has("header")) - dialect.setHasHeaderRow(jsonObj.getBoolean("header")); - if (jsonObj.has("lineTerminator")) - dialect.setLineTerminator(jsonObj.getString("lineTerminator")); - if (jsonObj.has("quoteChar")) - dialect.setQuoteChar(jsonObj.getString("quoteChar").charAt(0)); - if (jsonObj.has("doubleQuote")) - dialect.setDoubleQuote(jsonObj.getBoolean("doubleQuote")); - if (jsonObj.has("caseSensitiveHeader")) - dialect.setCaseSensitiveHeader(jsonObj.getBoolean("caseSensitiveHeader")); - if (jsonObj.has("csvddfVersion")) - dialect.setCsvddfVersion(jsonObj.getDouble("csvddfVersion")); - return dialect; + return JsonUtil.getInstance().deserialize(json, Dialect.class); } /** * Get JSON representation of the object. * @return a String representing the properties of this object encoded as JSON */ - public String getJson() { - return getJsonObject(true).toString(); - } - - private JSONObject getJsonObject(boolean checkDefault) { - JSONObject retVal = new JSONObject(); - retVal = setProperty(retVal, "delimiter", delimiter, checkDefault); - retVal = setProperty(retVal, "escapeChar", escapeChar, checkDefault); - retVal = setProperty(retVal, "skipInitialSpace", skipInitialSpace, checkDefault); - retVal = setProperty(retVal, "nullSequence", nullSequence, checkDefault); - retVal = setProperty(retVal, "commentChar", commentChar, checkDefault); - retVal = setProperty(retVal, "header", hasHeaderRow, checkDefault); - retVal = setProperty(retVal, "lineTerminator", lineTerminator, checkDefault); - retVal = setProperty(retVal, "quoteChar", quoteChar, checkDefault); - retVal = setProperty(retVal, "doubleQuote", doubleQuote, checkDefault); - retVal = setProperty(retVal, "caseSensitiveHeader", caseSensitiveHeader, checkDefault); - retVal = setProperty(retVal, "csvddfVersion", csvddfVersion, checkDefault); - return retVal; + @JsonIgnore + public String asJson() { + return getJsonNode(true).toString(); + } + + private JsonNode getJsonNode(boolean checkDefault) { + JsonNode json = JsonUtil.getInstance().createNode(this); + return json; } public void writeJson (File outputFile) throws IOException{ @@ -215,16 +241,28 @@ public void writeJson (File outputFile) throws IOException{ } public void writeJson (OutputStream output) throws IOException{ - try (BufferedWriter file = new BufferedWriter(new OutputStreamWriter(output))) { - file.write(this.getJson()); + try (BufferedWriter file = new BufferedWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8))) { + file.write(this.asJson()); + } + } + + + public void writeDialect(Path parentFilePath) throws IOException { + if (!Files.exists(parentFilePath)) { + Files.createDirectories(parentFilePath); + } + Files.deleteIfExists(parentFilePath); + try (Writer wr = Files.newBufferedWriter(parentFilePath, StandardCharsets.UTF_8)) { + wr.write(asJson()); } } + Object get(String key) { - JSONObject obj = getJsonObject(true); + JsonNode obj = getJsonNode(true); return obj.get(key); } - JSONObject setProperty(JSONObject obj, String key, Object value, boolean checkDefault) { + void setProperty(Map obj, String key, Object value, boolean checkDefault) { if (checkDefault) { if ((value != null) && (value != DEFAULT.get(key))) { obj.put(key, value); @@ -232,7 +270,6 @@ JSONObject setProperty(JSONObject obj, String key, Object value, boolean checkDe } else { obj.put(key, value); } - return obj; } public String getDelimiter() { @@ -263,6 +300,7 @@ public boolean isSkipInitialSpace() { return skipInitialSpace; } + @JsonProperty(value = "header") public boolean isHasHeaderRow() { return hasHeaderRow; } @@ -323,6 +361,16 @@ public void setCsvddfVersion(Double csvddfVersion) { this.csvddfVersion = csvddfVersion; } + @JsonAnyGetter + public Map getAdditionalProperties() { + return additionalProperties; + } + + @JsonAnySetter + public void setAdditionalProperties(Map additionalProperties) { + this.additionalProperties = additionalProperties; + } + public boolean equals(final Object o) { if (o == this) return true; if (!(o instanceof Dialect)) return false; @@ -398,4 +446,5 @@ public int hashCode() { result = result * PRIME + ($caseSensitiveHeader == null ? 43 : $caseSensitiveHeader.hashCode()); return result; } + } \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java index 547ca88..6f9dd8a 100644 --- a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java +++ b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java @@ -1,12 +1,22 @@ package io.frictionlessdata.datapackage; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ObjectNode; +import tools.jackson.databind.node.StringNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.datapackage.exceptions.DataPackageFileOrUrlNotFoundException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.datapackage.resource.Resource; +import io.frictionlessdata.tableschema.exception.JsonParsingException; +import io.frictionlessdata.tableschema.io.FileReference; +import io.frictionlessdata.tableschema.io.LocalFileReference; +import io.frictionlessdata.tableschema.io.URLFileReference; import io.frictionlessdata.tableschema.schema.Schema; -import io.frictionlessdata.tableschema.datasourceformats.DataSourceFormat; -import org.json.JSONArray; -import org.json.JSONObject; +import io.frictionlessdata.tableschema.util.JsonUtil; import java.io.*; import java.net.URL; @@ -22,12 +32,10 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import static io.frictionlessdata.datapackage.Package.isValidUrl; +import static io.frictionlessdata.datapackage.Validator.isValidUrl; -public abstract class JSONBase { - static final int JSON_INDENT_FACTOR = 4;// JSON keys. - // FIXME: Use somethign like GSON instead so this explicit mapping is not - // necessary? +@JsonInclude(value = Include.NON_EMPTY, content = Include.NON_EMPTY ) +public abstract class JSONBase implements BaseInterface { public static final String JSON_KEY_NAME = "name"; public static final String JSON_KEY_PROFILE = "profile"; public static final String JSON_KEY_PATH = "path"; @@ -47,33 +55,29 @@ public abstract class JSONBase { /** * If true, we are reading from an archive format, eg. ZIP */ + @JsonIgnore boolean isArchivePackage = false; // Metadata properties. // Required properties. - private String name; + protected String name; // Recommended properties. - private String profile = null; + protected String profile = null; // Optional properties. - private String title = null; - private String description = null; + protected String title = null; + protected String description = null; + protected String mediaType = null; + protected String encoding = null; + protected Integer bytes = null; + protected String hash = null; - String format = null; - private String mediaType = null; - private String encoding = null; - private Integer bytes = null; - private String hash = null; + private List sources = null; + private List licenses = null; - Dialect dialect; - private JSONArray sources = null; - private JSONArray licenses = null; - - // Schema - private Schema schema = null; - - protected Map originalReferences = new HashMap<>(); + @JsonIgnore + protected Map originalReferences = new HashMap<>(); /** * @return the name */ @@ -89,11 +93,6 @@ public abstract class JSONBase { */ public String getProfile(){return profile;} - /** - * @param profile the profile to set - */ - public void setProfile(String profile){this.profile = profile;} - /** * @return the title */ @@ -155,76 +154,99 @@ public abstract class JSONBase { */ public void setHash(String hash){this.hash = hash;} - public Schema getSchema(){return schema;} - - public void setSchema(Schema schema){this.schema = schema;} - - public String getSchemaReference() { - if (null == originalReferences.get(JSONBase.JSON_KEY_SCHEMA)) - return null; - return originalReferences.get(JSONBase.JSON_KEY_SCHEMA).toString(); - } - - public JSONArray getSources(){ + public List getSources(){ return sources; } - public void setSources(JSONArray sources){ + public void setSources(List sources){ this.sources = sources; } /** * @return the licenses */ - public JSONArray getLicenses(){return licenses;} + public List getLicenses(){return licenses;} /** * @param licenses the licenses to set */ - public void setLicenses(JSONArray licenses){this.licenses = licenses;} + public void setLicenses(List licenses){this.licenses = licenses;} + @JsonIgnore + public Map getOriginalReferences() { + return originalReferences; + } - public static Schema buildSchema(JSONObject resourceJson, Object basePath, boolean isArchivePackage) throws Exception { - // Get the schema and dereference it. Enables validation against it. - Object schemaObj = resourceJson.has(JSONBase.JSON_KEY_SCHEMA) ? resourceJson.get(JSONBase.JSON_KEY_SCHEMA) : null; - JSONObject dereferencedSchema = dereference(schemaObj, basePath, isArchivePackage); - if (null != dereferencedSchema) { - return Schema.fromJson(dereferencedSchema.toString(), false); + public static Schema buildSchema(JsonNode resourceJson, Object basePath, boolean isArchivePackage) + throws Exception { + FileReference ref = referenceFromJson(resourceJson, JSON_KEY_SCHEMA, basePath); + if (null != ref) { + return Schema.fromJson(ref, true); } - return null; + Object schemaObj = resourceJson.has(JSON_KEY_SCHEMA) + ? resourceJson.get(JSON_KEY_SCHEMA) + : null; + if (null == schemaObj) + return null; + return Schema.fromJson(dereference(schemaObj, basePath, isArchivePackage).toString(), true); } - public static Dialect buildDialect (JSONObject resourceJson, Object basePath, boolean isArchivePackage) throws Exception { - // Get the dialect and dereference it. Enables validation against it. - Object dialectObj = resourceJson.has(JSONBase.JSON_KEY_DIALECT) ? resourceJson.get(JSONBase.JSON_KEY_DIALECT) : null; - JSONObject dereferencedDialect = dereference(dialectObj, basePath, isArchivePackage); - if (null != dereferencedDialect) { - return Dialect.fromJson(dereferencedDialect.toString()); + public static Dialect buildDialect (JsonNode resourceJson, Object basePath, boolean isArchivePackage) + throws Exception { + FileReference ref = referenceFromJson(resourceJson, JSON_KEY_DIALECT, basePath); + if (null != ref) { + return Dialect.fromJson(ref); } - return null; + Object dialectObj = resourceJson.has(JSON_KEY_DIALECT) + ? resourceJson.get(JSON_KEY_DIALECT) + : null; + if (null == dialectObj) + return null; + return Dialect.fromJson(dereference(dialectObj, basePath, isArchivePackage).toString()); } - public static void setFromJson(JSONObject resourceJson, JSONBase retVal, Schema schema) { - if (resourceJson.has(JSONBase.JSON_KEY_SCHEMA)) - retVal.originalReferences.put(JSONBase.JSON_KEY_SCHEMA, resourceJson.get(JSONBase.JSON_KEY_SCHEMA)); - if (resourceJson.has(JSONBase.JSON_KEY_DIALECT)) - retVal.originalReferences.put(JSONBase.JSON_KEY_DIALECT, resourceJson.get(JSONBase.JSON_KEY_DIALECT)); - - //FIXME: Again, could be greatly simplified amd much more - // elegant if we use a library like GJSON... - String name = resourceJson.has(JSONBase.JSON_KEY_NAME) ? resourceJson.getString(JSONBase.JSON_KEY_NAME) : null; - String profile = resourceJson.has(JSONBase.JSON_KEY_PROFILE) ? resourceJson.getString(JSONBase.JSON_KEY_PROFILE) : null; - String title = resourceJson.has(JSONBase.JSON_KEY_TITLE) ? resourceJson.getString(JSONBase.JSON_KEY_TITLE) : null; - String description = resourceJson.has(JSONBase.JSON_KEY_DESCRIPTION) ? resourceJson.getString(JSONBase.JSON_KEY_DESCRIPTION) : null; - String mediaType = resourceJson.has(JSONBase.JSON_KEY_MEDIA_TYPE) ? resourceJson.getString(JSONBase.JSON_KEY_MEDIA_TYPE) : null; - String encoding = resourceJson.has(JSONBase.JSON_KEY_ENCODING) ? resourceJson.getString(JSONBase.JSON_KEY_ENCODING) : null; - Integer bytes = resourceJson.has(JSONBase.JSON_KEY_BYTES) ? resourceJson.getInt(JSONBase.JSON_KEY_BYTES) : null; - String hash = resourceJson.has(JSONBase.JSON_KEY_HASH) ? resourceJson.getString(JSONBase.JSON_KEY_HASH) : null; - - JSONArray sources = resourceJson.has(JSONBase.JSON_KEY_SOURCES) ? resourceJson.getJSONArray(JSONBase.JSON_KEY_SOURCES) : null; - JSONArray licenses = resourceJson.has(JSONBase.JSON_KEY_LICENSES) ? resourceJson.getJSONArray(JSONBase.JSON_KEY_LICENSES) : null; + private static FileReference referenceFromJson(JsonNode resourceJson, String key, Object basePath) + throws IOException { + Object dialectObj = resourceJson.has(key) + ? resourceJson.get(key) + : null; + if (null == dialectObj) + return null; + Object refObj = determineType(dialectObj, basePath); + FileReference ref = null; + if (refObj instanceof URL) { + ref = new URLFileReference((URL)refObj); + } else if (refObj instanceof File) { + ref = new LocalFileReference(((Path)basePath).toFile(), dialectObj.toString()); + } + return ref; + } + + public static void setFromJson(JsonNode resourceJson, JSONBase retVal) { + if (resourceJson.has(JSONBase.JSON_KEY_SCHEMA) && resourceJson.get(JSONBase.JSON_KEY_SCHEMA).isString()) + retVal.originalReferences.put(JSONBase.JSON_KEY_SCHEMA, resourceJson.get(JSONBase.JSON_KEY_SCHEMA).asString()); + if (resourceJson.has(JSONBase.JSON_KEY_DIALECT) && resourceJson.get(JSONBase.JSON_KEY_DIALECT).isString()) + retVal.originalReferences.put(JSONBase.JSON_KEY_DIALECT, resourceJson.get(JSONBase.JSON_KEY_DIALECT).asString()); + + // TODO: A mapper library might be useful, but not required + String name = textValueOrNull(resourceJson, JSONBase.JSON_KEY_NAME); + String profile = textValueOrNull(resourceJson, JSONBase.JSON_KEY_PROFILE); + String title = textValueOrNull(resourceJson, JSONBase.JSON_KEY_TITLE); + String description = textValueOrNull(resourceJson, JSONBase.JSON_KEY_DESCRIPTION); + String mediaType = textValueOrNull(resourceJson, JSONBase.JSON_KEY_MEDIA_TYPE); + String encoding = textValueOrNull(resourceJson, JSONBase.JSON_KEY_ENCODING); + Integer bytes = resourceJson.has(JSONBase.JSON_KEY_BYTES) ? resourceJson.get(JSONBase.JSON_KEY_BYTES).asInt() : null; + String hash = textValueOrNull(resourceJson, JSONBase.JSON_KEY_HASH); + + List sources = null; + if(resourceJson.has(JSONBase.JSON_KEY_SOURCES) && resourceJson.get(JSON_KEY_SOURCES).isArray()) { + sources = JsonUtil.getInstance().deserialize(resourceJson.get(JSONBase.JSON_KEY_SOURCES), new TypeReference<>() {}); + } + List licenses = null; + if(resourceJson.has(JSONBase.JSON_KEY_LICENSES) && resourceJson.get(JSONBase.JSON_KEY_LICENSES).isArray()){ + licenses = JsonUtil.getInstance().deserialize(resourceJson.get(JSONBase.JSON_KEY_LICENSES), new TypeReference<>() {}); + } retVal.setName(name); - retVal.setSchema(schema); retVal.setProfile(profile); retVal.setTitle(title); retVal.setDescription(description); @@ -236,6 +258,10 @@ public static void setFromJson(JSONObject resourceJson, JSONBase retVal, Schema retVal.setLicenses(licenses); } + private static String textValueOrNull(JsonNode source, String fieldName) { + return source.has(fieldName) ? source.get(fieldName).asString() : null; + } + static String getFileContentAsString(InputStream stream) { try (BufferedReader rdr = new BufferedReader( @@ -253,6 +279,8 @@ static String getFileContentAsString(URL url) { try { return getFileContentAsString(url.openStream()); } catch (Exception ex) { + if (ex instanceof FileNotFoundException) + throw new DataPackageValidationException(ex.getMessage(), ex); throw new DataPackageException(ex); } } @@ -308,30 +336,56 @@ private static ZipEntry findZipEntry(ZipFile zipFile, String fileName) { return null; } - protected static String getZipFileContentAsString(Path inFilePath, String fileName) throws IOException { + protected static String getZipFileContentAsString(Path inFilePath, String fileName, boolean strict) throws IOException { // Read in memory the file inside the zip. ZipFile zipFile = new ZipFile(inFilePath.toFile()); ZipEntry entry = findZipEntry(zipFile, fileName); - // Throw exception if expected datapackage.json file not found. - if(entry == null){ - throw new DataPackageException("The zip file does not contain the expected file: " + fileName); + if (entry == null) { + if (strict) { + throw new DataPackageException("The zip file does not contain the expected file: " + fileName); + } else { + return null; + } } - // Read the datapackage.json file inside the zip + String content; try (InputStream stream = zipFile.getInputStream(entry)) { - return getFileContentAsString(stream); + content = getFileContentAsString(stream); + } + zipFile.close(); + return content; + } + + protected static byte[] getZipFileContentAsByteArray(Path inFilePath, String fileName, boolean strict) throws IOException { + // Read in memory the file inside the zip. + ZipFile zipFile = new ZipFile(inFilePath.toFile()); + ZipEntry entry = findZipEntry(zipFile, fileName); + + if (entry == null) { + if (strict) { + throw new DataPackageException("The zip file does not contain the expected file: " + fileName); + } else { + return null; + } + } + try (InputStream inputStream = zipFile.getInputStream(entry); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (int b; (b = inputStream.read()) != -1; ) { + out.write(b); + } + return out.toByteArray(); } } - public static JSONObject dereference(File fileObj, Path basePath, boolean isArchivePackage) throws IOException { + public static ObjectNode dereference(File fileObj, Path basePath, boolean isArchivePackage) throws IOException { String jsonContentString; if (isArchivePackage) { String filePath = fileObj.getPath(); if (File.separator.equals("\\")) { filePath = filePath.replaceAll("\\\\", "/"); } - jsonContentString = getZipFileContentAsString(basePath, filePath); + jsonContentString = getZipFileContentAsString(basePath, filePath, true); } else { /* If reference is file path. from the spec: "SECURITY: / (absolute path) and ../ (relative parent path) @@ -348,7 +402,7 @@ public static JSONObject dereference(File fileObj, Path basePath, boolean isArch throw new DataPackageFileOrUrlNotFoundException("Local file not found: " + fileObj); } } - return new JSONObject(jsonContentString); + return (ObjectNode) createNode(jsonContentString); } /** @@ -357,46 +411,48 @@ public static JSONObject dereference(File fileObj, Path basePath, boolean isArch * from the URL content * @param url fully qualified URL or path fragment * @param basePath base URL, only used if we are dealing with a path fragment - * @return a JSONObject built from the url content + * @return a JsonNode built from the url content * @throws IOException if fetching the contents of the URL goes wrong */ - private static JSONObject dereference(String url, URL basePath) throws IOException { - JSONObject dereferencedObj = null; + private static ObjectNode dereference(String url, URL basePath) throws IOException { + JsonNode dereferencedObj; if (isValidUrl(url)) { // Create the dereferenced object from the remote file. String jsonContentString = getFileContentAsString(new URL(url)); - dereferencedObj = new JSONObject(jsonContentString); + dereferencedObj = createNode(jsonContentString); } else { URL lURL = new URL(basePath.toExternalForm()+url); if (isValidUrl(lURL)) { String jsonContentString = getFileContentAsString(lURL); - dereferencedObj = new JSONObject(jsonContentString); + dereferencedObj = createNode(jsonContentString); } else { throw new DataPackageFileOrUrlNotFoundException("URL not found"+lURL); } } - return dereferencedObj; + return (ObjectNode) dereferencedObj; } - public static JSONObject dereference(Object obj, Object basePath, boolean isArchivePackage) throws IOException { + public static ObjectNode dereference(Object obj, Object basePath, boolean isArchivePackage) throws IOException { if (null == obj) return null; // Object is already a dereferenced object. - if(obj instanceof JSONObject){ + else if(obj instanceof ObjectNode){ // Don't need to do anything, just cast and return. - return (JSONObject)obj; + return (ObjectNode)obj; + } else if (obj instanceof StringNode) { + return dereference(((StringNode) obj).asString(), basePath, isArchivePackage); } else if(obj instanceof String){ String reference = (String)obj; if (isValidUrl(reference)) if (basePath instanceof File) { String jsonString = getFileContentAsString(new URL(reference)); - return new JSONObject(jsonString); + return (ObjectNode) createNode(jsonString); } else { String jsonString = getFileContentAsString(new URL(reference)); - return new JSONObject(jsonString); + return (ObjectNode) createNode(jsonString); } else if (basePath instanceof URL) { return dereference(reference, (URL) basePath); @@ -406,4 +462,34 @@ else if (basePath instanceof URL) { return null; } + + protected static JsonNode createNode(String json) { + try { + return JsonUtil.getInstance().createNode(json); + } catch (JsonParsingException ex) { + throw new DataPackageException(ex); + } + } + + public static Object determineType(Object obj, Object basePath) throws IOException { + if (null == obj) + return null; + // Object is already a dereferenced object. + if(obj instanceof JsonNode){ + // Don't need to do anything, just cast and return. + return obj; + } else if(obj instanceof String){ + String reference = (String)obj; + if (isValidUrl(reference)){ + return new URL(reference); + } + else if (basePath instanceof URL) { + return new URL(((URL)basePath), reference); + } else { + return new File(((Path)basePath).toFile(), reference); + } + } + + return null; + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/License.java b/src/main/java/io/frictionlessdata/datapackage/License.java new file mode 100644 index 0000000..5167c65 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/License.java @@ -0,0 +1,33 @@ +package io.frictionlessdata.datapackage; + +public class License { + private String name; + + private String path; + + private String title; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } +} diff --git a/src/main/java/io/frictionlessdata/datapackage/Package.java b/src/main/java/io/frictionlessdata/datapackage/Package.java index 49a526c..c49343b 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Package.java +++ b/src/main/java/io/frictionlessdata/datapackage/Package.java @@ -1,39 +1,53 @@ package io.frictionlessdata.datapackage; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import io.frictionlessdata.datapackage.resource.*; -import io.frictionlessdata.tableschema.schema.Schema; -import io.frictionlessdata.tableschema.Table; -import org.apache.commons.collections.list.UnmodifiableList; -import org.apache.commons.collections.set.UnmodifiableSet; +import io.frictionlessdata.datapackage.exceptions.DataPackageFileOrUrlNotFoundException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.datapackage.resource.AbstractDataResource; +import io.frictionlessdata.datapackage.resource.AbstractReferencebasedResource; +import io.frictionlessdata.datapackage.resource.Resource; +import io.frictionlessdata.tableschema.exception.JsonParsingException; +import io.frictionlessdata.tableschema.exception.ValidationException; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.collections4.list.UnmodifiableList; +import org.apache.commons.collections4.set.UnmodifiableSet; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.validator.routines.UrlValidator; -import org.everit.json.schema.ValidationException; -import org.json.JSONArray; -import org.json.JSONObject; import java.io.*; +import java.math.BigDecimal; +import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.*; import java.time.ZonedDateTime; import java.util.*; +import java.util.function.Consumer; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; + +import static io.frictionlessdata.datapackage.Validator.isValidUrl; /** * Load, validate, create, and save a datapackage object according to the specifications at - * https://github.com/frictionlessdata/specs/blob/master/specs/data-package.md + * https://specs.frictionlessdata.io/data-package */ +@JsonInclude(value = Include.NON_EMPTY, content = Include.NON_EMPTY ) public class Package extends JSONBase{ - public static final String DATAPACKAGE_FILENAME = "datapackage.json"; + public static final String DATAPACKAGE_FILENAME = "datapackage.json"; private static final String JSON_KEY_RESOURCES = "resources"; private static final String JSON_KEY_ID = "id"; private static final String JSON_KEY_VERSION = "version"; @@ -43,22 +57,33 @@ public class Package extends JSONBase{ private static final String JSON_KEY_CREATED = "created"; private static final String JSON_KEY_CONTRIBUTORS = "contributors"; + private static final List wellKnownKeys = Arrays.asList( + JSON_KEY_NAME, JSON_KEY_RESOURCES, JSON_KEY_ID, JSON_KEY_VERSION, + JSON_KEY_HOMEPAGE, JSON_KEY_IMAGE, JSON_KEY_CREATED, JSON_KEY_CONTRIBUTORS, + JSON_KEY_KEYWORDS, JSONBase.JSON_KEY_SCHEMA, JSONBase.JSON_KEY_NAME, JSONBase.JSON_KEY_DATA, + JSONBase.JSON_KEY_DIALECT, JSONBase.JSON_KEY_LICENSES, JSONBase.JSON_KEY_SOURCES, JSONBase.JSON_KEY_PROFILE); + + /** + * The charset (encoding) for writing + */ + @JsonIgnore + private final Charset charset = StandardCharsets.UTF_8; + + // Filesystem path pointing to the Package; either ZIP file or directory private Object basePath = null; private String id; private String version; - private int[] versionParts; private URL homepage; - private Set keywords = new TreeSet<>(); + private Set keywords = new LinkedHashSet<>(); private String image; + private byte[] imageData; private ZonedDateTime created; private List contributors = new ArrayList<>(); - private Map otherProperties = new LinkedHashMap<>(); - private JSONObject jsonObject = new JSONObject(); + private ObjectNode jsonObject = JsonUtil.getInstance().createNode(); private boolean strictValidation = false; - private List resources = new ArrayList<>(); - private List errors = new ArrayList<>(); - private Validator validator = new Validator(); + private final List resources = new ArrayList<>(); + private final List errors = new ArrayList<>(); /** * Create a new DataPackage and initialize with a number of Resources. @@ -70,16 +95,18 @@ public class Package extends JSONBase{ */ public Package(Collection resources) throws IOException { for (Resource r : resources) { - addResource(r); - } + addResource(r, false); + } UUID uuid = UUID.randomUUID(); id = uuid.toString(); validate(); } /** - * Load from String representation of JSON object. To prevent file system traversal attacks, - * the basePath must be explicitly set here, the `basePath` variable cannot be null. + * Load from String representation of JSON object. The resources of the package could be either inline JSON + * or relative path references to files.To prevent file system traversal attacks + * while loading Resources, the basePath must be explicitly set here, the `basePath` + * variable cannot be null. * * The basePath is the path that is used as a jail root for Resource creation - * no absolute paths for Resources are allowed, they must all be relative to and @@ -99,11 +126,13 @@ public Package(String jsonStringSource, Path basePath, boolean strict) throws Ex throw new DataPackageException("basePath cannot be null for JSON-based DataPackages "); this.basePath = basePath; - // Create and set the JSONObject fpr the String representation of desriptor JSON object. - this.setJson(new JSONObject(jsonStringSource)); + // Create and set the JsonNode for the String representation of descriptor JSON object. + try { + this.setJson((ObjectNode) JsonUtil.getInstance().createNode(jsonStringSource)); + } catch(JsonParsingException ex) { + throw new DataPackageException(ex.getMessage(), ex); + } - // If String representation of desriptor JSON object is provided. - this.validate(); } /** @@ -133,10 +162,14 @@ public Package(URL urlSource, boolean strict) throws Exception { // Get string content of given remove file. String jsonString = getFileContentAsString(urlSource); - // Create JSONObject and validate. - this.setJson(new JSONObject(jsonString)); - this.validate(); - + // Create JsonNode and validate. + try { + this.setJson((ObjectNode) createNode(jsonString)); + } catch (DataPackageException ex) { + if (strict) { + throw ex; + } + } } /** @@ -159,59 +192,346 @@ public Package(URL urlSource, boolean strict) throws Exception { * * @param descriptorFile local file path that points to the DataPackage Descriptor (if it's in * a local directory) or the ZIP file if it's a ZIP-based - * package. + * package or to the parent directory * @param strict whether to use strict schema parsing + * @throws DataPackageFileOrUrlNotFoundException if the path is invalid * @throws IOException thrown if I/O operations fail * @throws DataPackageException thrown if constructing the Descriptor fails */ public Package(Path descriptorFile, boolean strict) throws Exception { this.strictValidation = strict; - JSONObject sourceJsonObject; - if (descriptorFile.toFile().getName().toLowerCase().endsWith(".zip")) { - isArchivePackage = true; + JsonNode sourceJsonNode; + if (!descriptorFile.toFile().exists()) { + throw new DataPackageFileOrUrlNotFoundException("File " + descriptorFile + "does not exist"); + } + if (descriptorFile.toFile().isDirectory()) { basePath = descriptorFile; - sourceJsonObject = new JSONObject(JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME)); + File realDescriptor = new File(descriptorFile.toFile(), DATAPACKAGE_FILENAME); + String sourceJsonString = getFileContentAsString(realDescriptor.toPath()); + sourceJsonNode = createNode(sourceJsonString); } else { - basePath = descriptorFile.getParent(); - String sourceJsonString = getFileContentAsString(descriptorFile); - sourceJsonObject = new JSONObject(sourceJsonString); + if (isArchive(descriptorFile.toFile())) { + isArchivePackage = true; + basePath = descriptorFile; + sourceJsonNode = createNode(JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME, true)); + } else { + basePath = descriptorFile.getParent(); + String sourceJsonString = getFileContentAsString(descriptorFile); + sourceJsonNode = createNode(sourceJsonString); + } } - this.setJson(sourceJsonObject); - this.validate(); + this.setJson((ObjectNode) sourceJsonNode); } + public Resource getResource(String resourceName){ + for (Resource resource : this.resources) { + if (resource.getName().equalsIgnoreCase(resourceName)) { + return resource; + } + } + return null; + } - private FileSystem getTargetFileSystem(File outputDir, boolean zipCompressed) throws IOException { - FileSystem outFs = null; - if (zipCompressed) { - if (outputDir.exists()) { - throw new DataPackageException("Cannot save into existing ZIP file: " - +outputDir.getName()); + /** + * Return a List of data {@link Resource}s of the Package. See https://specs.frictionlessdata.io/data-resource/ + * for details + * + * @return the resources as a List. + */ + public List getResources(){ + return new ArrayList<>(this.resources); + } + + /** + * Return a List of all Resources. + * + * @return the resource names as a List. + */ + @JsonIgnore + public List getResourceNames(){ + return resources.stream().map(Resource::getName).collect(Collectors.toList()); + } + + /** + * Return the profile of the Package descriptor. See https://specs.frictionlessdata.io/profiles/ + * for details + * + * @return the profile. + */ + @Override + public String getProfile() { + if (null == super.getProfile()) + return Profile.PROFILE_DATA_PACKAGE_DEFAULT; + return super.getProfile(); + } + + public String getId() { + return id; + } + + @JsonProperty("keywords") + public Set getKeywords() { + if (null == keywords) + return null; + return UnmodifiableSet.unmodifiableSet(keywords); + } + + @JsonProperty("version") + public String getVersion() { + return version; + } + + @JsonProperty("homepage") + public URL getHomepage() { + return homepage; + } + + /** + * Returns the path or URL for the image according to the spec: + * https://specs.frictionlessdata.io/data-package/#image + * + * @return path or URL to the image data + */ + @JsonProperty("image") + public String getImagePath() { + return image; + } + + /** + * Returns the image data if the image is stored inside the data package, null if {@link #getImagePath()} + * would return a URL. + * + * If the referenced file does not exist in the package, return null + * + * @return binary image data + */ + @JsonIgnore + public byte[] getImage() throws IOException { + if (null != imageData) + return imageData; + if (!StringUtils.isEmpty(image)) { + if (isArchivePackage) { + return getZipFileContentAsByteArray((Path)basePath, image, false); + } else { + File imgFile = new File (((Path)basePath).toFile(), image); + try (InputStream inputStream = new FileInputStream(imgFile); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (int b; (b = inputStream.read()) != -1; ) { + out.write(b); + } + return out.toByteArray(); + } } - Map env = new HashMap<>(); - env.put("create", "true"); - outFs = FileSystems.newFileSystem(URI.create("jar:" + outputDir.toURI().toString()), env); + } + return null; + } + + public ZonedDateTime getCreated() { + return created; + } + + + public List getContributors() { + if (null == contributors) + return null; + return UnmodifiableList.unmodifiableList(contributors); + } + + /** + * Get the value of a named property of the Package (the `datapackage.json`). + * @return a Java class, either a string, BigInteger, BitDecimal, array or an object + */ + public Object getProperty(String key) { + if (!this.jsonObject.has(key)) { + return null; + } + JsonNode jNode = jsonObject.get(key); + if (jNode.isArray()) { + return getProperty(key, new TypeReference>() {}); + } else if (jNode.isString()) { + return getProperty(key, new TypeReference() {}); + } else if (jNode.isBoolean()) { + return getProperty(key, new TypeReference() {}); + } else if (jNode.isFloatingPointNumber()) { + return getProperty(key, new TypeReference() {}); + } else if (jNode.isIntegralNumber()) { + return getProperty(key, new TypeReference() {}); + } else if (jNode.isObject()) { + return getProperty(key, new TypeReference() {}); + } else if (jNode.isNull() || jNode.isEmpty() || jNode.isMissingNode()) { + return null; + } + return null; + } + + /** + * Get the value of a Package property (i.e. from the `datapackage.json`. The value will be returned + * as a Java Object corresponding to `typeRef` + * + * @param key the property name + * @param typeRef the Java type of the property value. + */ + public Object getProperty(String key, TypeReference typeRef) { + if (!this.jsonObject.has(key)) { + return null; + } + JsonNode jNode = jsonObject.get(key); + return JsonUtil.getInstance().deserialize(jNode, typeRef); + } + + /** + * Convert both the descriptor and all linked Resources to JSON and return them. + * @return JSON-String representation of the Package + */ + @JsonIgnore + public String asJson(){ + return getJsonNode().toPrettyString(); + } + + /** + * Convert both the descriptor and all linked Resources to JSON and return them. + * @return JSON-String representation of the Package + * + * Deprecated, use {@link #asJson()} instead. + */ + @Deprecated + @JsonIgnore + public String getJson(){ + return getJsonNode().toPrettyString(); + } + + /** + * Get the value of a Package property (i.e. from the `datapackage.json`. The value will be returned + * as a Java Object corresponding to `clazz` + * + * @param key the property name + * @param clazz the Java type of the property value. + */ + public Object getProperty(String key, Class clazz) { + if (!this.jsonObject.has(key)) { + return null; + } + JsonNode jNode = jsonObject.get(key); + return JsonUtil.getInstance().deserialize(jNode, clazz); + } + + /** + * Returns the validation status of this Data Package. Always `true` if strict mode is enabled because reading + * an invalid Package would throw an exception. + * @return true if either `strictValidation` is true or no errors were encountered + */ + @JsonIgnore + public boolean isValid() { + if (strictValidation){ + return true; } else { - if (!(outputDir.isDirectory())) { - throw new DataPackageException("Target for save() exists and is a regular file: " - +outputDir.getName()); + return errors.isEmpty(); + } + } + + public void addContributor (Contributor contributor) { + if (null == contributor) + return; + if (null == contributors) + contributors = new ArrayList<>(); + this.contributors.add(contributor); + } + + /** + * Add a {@link Resource}s to the Package. The Resource will be validated and a {@link ValidationException} is + * thrown if it is invalid + */ + public void addResource(Resource resource) + throws IOException, ValidationException, DataPackageException{ + addResource(resource, true); + } + + + public void setId(String id) { + this.id = id; + } + + public void setKeywords(Set keywords) { + if (null == keywords) + return; + this.keywords = new LinkedHashSet<>(keywords); + } + + /** + * @param profile the profile to set + */ + public void setProfile(String profile){ + if (null != profile) { + if ((profile.equals(Profile.PROFILE_DATA_RESOURCE_DEFAULT)) + || (profile.equals(Profile.PROFILE_TABULAR_DATA_RESOURCE))) { + throw new DataPackageValidationException("Cannot set profile " + profile + " on a data package"); } - outFs = outputDir.toPath().getFileSystem(); } - return outFs; + this.profile = profile; } - private void writeDescriptor (FileSystem outFs, String parentDirName) throws IOException { - Path nf = outFs.getPath(parentDirName+File.separator+DATAPACKAGE_FILENAME); - try (Writer writer = Files.newBufferedWriter(nf, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) { - writer.write(this.getJsonObject().toString(JSON_INDENT_FACTOR)); + /** + * Set a property and value on the Package. The value will be converted to a JsonObject and added to the + * datapackage.json on serialization + * @param key the property name + * @param value the value to set. + */ + public void setProperty(String key, Object value) { + this.jsonObject.set(key, JsonUtil.getInstance().createNode(value)); + } + + public void setProperty(String key, JsonNode value) throws DataPackageException{ + this.jsonObject.set(key, value); + } + + /** + * Set a number of properties at once. The `mapping` holds the properties as + * key/value pairs + * @param mapping the key/value map holding the properties + */ + public void setProperties(Map mapping) { + JsonUtil jsonUtil = JsonUtil.getInstance(); + for (String key : mapping.keySet()) { + JsonNode vNode = jsonUtil.createNode(mapping.get(key)); + jsonObject.set(key, vNode); + } + } + + /** + * Remove a {@link Resource}s from the Package. If no resource with a name matching `name`, no exception is thrown + */ + public void removeResource(String name){ + this.resources.removeIf(resource -> resource.getName().equalsIgnoreCase(name)); + } + + public void removeContributor (Contributor contributor) { + if (null == contributor) + return; + if (null == contributors) + return; + if (contributors.contains(contributor)) { + this.contributors.remove(contributor); + } + } + + public void removeProperty(String key){ + this.getJsonNode().remove(key); + } + + public void removeKeyword (String keyword) { + if (null == keyword) + return; + if (null == keywords) + return; + if (keywords.contains(keyword)) { + this.keywords.remove(keyword); } } /** - * Convert all Resources to CSV files, no matter whether they come from - * URLs, JSON Arrays, or files originally. The result is just one JSON - * file + * Convert all Resources to JSON, no matter whether they come from + * URLs, JSON Arrays, or files originally. Then write the descriptor and all Resources to file. + * The result is just one JSON file * @param outputDir the directory or ZIP file to write the "datapackage.json" * file to * @param zipCompressed whether we are writing to a ZIP archive @@ -226,7 +546,7 @@ public void writeFullyInlined (File outputDir, boolean zipCompressed) throws Exc writeDescriptor(outFs, parentDirName); for (Resource r : this.resources) { - r.writeDataAsCsv(outFs.getPath(parentDirName+File.separator), dialect); + r.writeData(outFs.getPath(parentDirName+File.separator)); } } @@ -239,230 +559,143 @@ public void writeFullyInlined (File outputDir, boolean zipCompressed) throws Exc * @throws Exception thrown if something goes wrong writing */ public void write (File outputDir, boolean zipCompressed) throws Exception { - FileSystem outFs = getTargetFileSystem(outputDir, zipCompressed); + write (outputDir, null, zipCompressed); + } + + private File findOrCreateOutputDir(File outputDir, boolean zipCompressed) { + if (!zipCompressed) { + String fName = outputDir.getName().toLowerCase(); + if ((fName.endsWith(".zip") || (fName.endsWith(".json")))) { + outputDir = outputDir.getParentFile(); + } + if (!outputDir.exists()) { + outputDir.mkdirs(); + } + } + return outputDir; + } + + /** + * Write this datapackage to an output directory or ZIP file. Creates at least a + * datapackage.json file and if this Package object holds file-based + * resources, dialect, or schemas, creates them as files. Takes a Consumer-function to allow + * for usecase-specific manipulation of the package contents. + * + * @param outputDir the directory or ZIP file to write the files to + * @param zipCompressed whether we are writing to a ZIP archive + * @param callback Callback interface that can be used to manipulate the Package contents after Resources and + * Descriptor have been written. + * @throws Exception thrown if something goes wrong writing + */ + public void write (File outputDir, Consumer callback, boolean zipCompressed) throws Exception { + this.isArchivePackage = zipCompressed; + File outDir = findOrCreateOutputDir(outputDir, zipCompressed); + FileSystem outFs = getTargetFileSystem(outDir, zipCompressed); String parentDirName = ""; if (!zipCompressed) { - parentDirName = outputDir.getPath(); + parentDirName = outDir.getPath(); } - writeDescriptor(outFs, parentDirName); // only file-based Resources need to be written to the DataPackage, URLs stay as // external references and JSONArray-based Resources got serialized as part of the // Descriptor file final List resourceList = resources .stream() - .filter((r) -> (r instanceof FilebasedResource)) + .filter(Resource::shouldSerializeToFile) .collect(Collectors.toList()); for (Resource r : resourceList) { - r.writeDataAsCsv(outFs.getPath(parentDirName), dialect); - String schemaRef = r.getSchemaReference(); - // write out schema file only if not null or URL - if ((null != schemaRef) && (!isValidUrl(schemaRef))) { - // URL fragments will not be written to disk either - if (!(r instanceof URLbasedResource)) { - Path schemaP = outFs.getPath(parentDirName+File.separator+schemaRef); - writeSchema(schemaP, r.getSchema()); - } + r.writeData(outFs.getPath(parentDirName )); + r.writeSchema(outFs.getPath(parentDirName)); + r.writeDialect(outFs.getPath(parentDirName)); + } + writeDescriptor(outFs, parentDirName); + + if (null != imageData) { + if (null != getBaseUrl()) { + throw new DataPackageException("Cannot add image data to a package read from an URL"); } - String dialectRef = r.getDialectReference(); - // write out schema file only if not null or URL - if ((null != dialectRef) && (!isValidUrl(dialectRef))) { - // URL fragments will not be written to disk either - if (!(r instanceof URLbasedResource)) { - Path dialectP = outFs.getPath(parentDirName+File.separator+dialectRef); - writeDialect(dialectP, r.getDialect()); + String fileName = (!StringUtils.isEmpty(this.image)) ? this.image : "image-file"; + String sanitizedFileName = fileName.replaceAll("[\\s/\\\\]+", "_"); + if (zipCompressed) { + Path imagePath = outFs.getPath(sanitizedFileName); + OutputStream out = Files.newOutputStream(imagePath); + out.write(imageData); + out.close(); + } else { + Path path = outFs.getPath(parentDirName); + File imageFile = new File(path.toFile(), sanitizedFileName); + try (FileOutputStream out = new FileOutputStream(imageFile)){ + out.write(imageData); } } } - // ZIP-FS needs close, but WindowsFileSystem unsurprisingly doesn't - // like to get closed... + + if (null != callback) { + callback.accept(outFs.getPath(parentDirName)); + } + + /* ZIP-FS needs close, but WindowsFileSystem unsurprisingly doesn't + like to get closed... + The empty catch block is intentional. + */ try { outFs.close(); } catch (UnsupportedOperationException es) {}; } + /** + * Serialize the whole package including Resources to JSON and write to a file + * + * @param outputFile File to write to + * @throws IOException if writing fails + */ public void writeJson (File outputFile) throws IOException{ try (FileOutputStream fos = new FileOutputStream(outputFile)) { writeJson(fos); } } + /** + * Serialize the whole package including Resources to JSON and write to an OutputStream + * + * @param output OutputStream to write to + * @throws IOException if writing fails + */ public void writeJson (OutputStream output) throws IOException{ - try (BufferedWriter file = new BufferedWriter(new OutputStreamWriter(output))) { - file.write(this.getJsonObject().toString(JSON_INDENT_FACTOR)); + try (BufferedWriter file = new BufferedWriter(new OutputStreamWriter(output, charset))) { + file.write(this.getJsonNode().toPrettyString()); } } - private void saveZip(File outputFilePath) throws IOException, DataPackageException{ - try(FileOutputStream fos = new FileOutputStream(outputFilePath)){ - try(BufferedOutputStream bos = new BufferedOutputStream(fos)){ - try(ZipOutputStream zos = new ZipOutputStream(bos)){ - // File is not on the disk, test.txt indicates - // only the file name to be put into the zip. - ZipEntry entry = new ZipEntry(DATAPACKAGE_FILENAME); - - zos.putNextEntry(entry); - zos.write(this.getJsonObject().toString(JSON_INDENT_FACTOR).getBytes()); - zos.closeEntry(); - } + /** + * Validation is strict or lenient depending on how the package was + * instantiated with the strict flag. + * @throws IOException if something goes wrong reading the datapackage + * @throws DataPackageException if validation fails and validation is strict + */ + final void validate() throws IOException, DataPackageException{ + try{ + ObjectNode jsonNode = this.getJsonNode(); + for (Resource r : this.getResources()) { + r.validate(this); + } + Validator.validate(jsonNode); + } catch(ValidationException | DataPackageException ve){ + if (this.strictValidation){ + throw ve; + }else{ + if (ve instanceof DataPackageValidationException) + errors.add((DataPackageValidationException)ve); + else + errors.add(new DataPackageValidationException (ve)); } } } - // TODO migrate into Schema.java - private static void writeSchema(Path parentFilePath, Schema schema) throws IOException { - if (!Files.exists(parentFilePath)) { - Files.createDirectories(parentFilePath); - } - Files.deleteIfExists(parentFilePath); - try (Writer wr = Files.newBufferedWriter(parentFilePath, StandardCharsets.UTF_8)) { - wr.write(schema.getJson()); - } - } - // TODO migrate into Dialet.java - private static void writeDialect(Path parentFilePath, Dialect dialect) throws IOException { - if (!Files.exists(parentFilePath)) { - Files.createDirectories(parentFilePath); - } - Files.deleteIfExists(parentFilePath); - try (Writer wr = Files.newBufferedWriter(parentFilePath, StandardCharsets.UTF_8)) { - wr.write(dialect.getJson()); - } - } - - public Resource getResource(String resourceName){ - for (Resource resource : this.resources) { - if (resource.getName().equalsIgnoreCase(resourceName)) { - return resource; - } - } - return null; - } - - - public List getResources(){ - return this.resources; - } - - private void validate(DataPackageException dpe) throws IOException { - if (dpe != null) { - if (this.strictValidation) { - throw dpe; - } else { - errors.add(dpe); - } - } - - // Validate. - this.validate(); - } - - private DataPackageException checkDuplicates(Resource resource) { - DataPackageException dpe = null; - // Check if there is duplication. - for (Resource value : this.resources) { - if (value.getName().equalsIgnoreCase(resource.getName())) { - dpe = new DataPackageException( - "A resource with the same name already exists."); - } - } - return dpe; - } - - public void addResource(Resource resource) - throws IOException, ValidationException, DataPackageException{ - DataPackageException dpe = null; - if (resource.getName() == null){ - dpe = new DataPackageException("Invalid Resource, it does not have a name property."); - } - if (resource instanceof AbstractDataResource) - addResource((AbstractDataResource) resource); - else if (resource instanceof AbstractReferencebasedResource) - addResource((AbstractReferencebasedResource) resource); - validate(dpe); - } - - void addResource(AbstractDataResource resource) - throws IOException, ValidationException, DataPackageException{ - DataPackageException dpe = null; - // If a name property isn't given... - if ((resource).getData() == null || (resource).getFormat() == null) { - dpe = new DataPackageException("Invalid Resource. The data and format properties cannot be null."); - } else { - dpe = checkDuplicates(resource); - } - validate(dpe); - - this.resources.add(resource); - } - - void addResource(AbstractReferencebasedResource resource) - throws IOException, ValidationException, DataPackageException{ - DataPackageException dpe = null; - if (resource.getPaths() == null) { - dpe = new DataPackageException("Invalid Resource. The path property cannot be null."); - } else { - dpe = checkDuplicates(resource); - } - validate(dpe); - - this.resources.add(resource); - } - - void removeResource(String name){ - this.resources.removeIf(resource -> resource.getName().equalsIgnoreCase(name)); - } - - - void addProperty(String key, String value) throws DataPackageException{ - if(this.getJsonObject().has(key)){ - throw new DataPackageException("A property with the same key already exists."); - }else{ - this.getJsonObject().put(key, value); - } - } - - public void addProperty(String key, JSONObject value) throws DataPackageException{ - if(this.getJsonObject().has(key)){ - throw new DataPackageException("A property with the same key already exists."); - }else{ - this.getJsonObject().put(key, value); - } - } - - public void addProperty(String key, JSONArray value) throws DataPackageException{ - if(this.getJsonObject().has(key)){ - throw new DataPackageException("A property with the same key already exists."); - }else{ - this.getJsonObject().put(key, value); - } - } - - public void removeProperty(String key){ - this.getJsonObject().remove(key); - } - - /** - * Validation is strict or unstrict depending on how the package was - * instantiated with the strict flag. - * @throws IOException - * @throws DataPackageException - */ - final void validate() throws IOException, DataPackageException{ - try{ - this.validator.validate(this.getJson()); - }catch(ValidationException ve){ - if(this.strictValidation){ - throw ve; - }else{ - errors.add(ve); - } - } - } - + @JsonIgnore final Path getBasePath(){ if (basePath instanceof File) return ((File)this.basePath).toPath(); @@ -472,82 +705,97 @@ else if (basePath instanceof Path) return null; } - + @JsonIgnore final URL getBaseUrl(){ if (basePath instanceof URL) return (URL)basePath; return null; } - /** - * Convert both the descriptor and all linked Resources to JSON and return them. - * @return - */ - public String getJson(){ - Iterator resourceIter = resources.iterator(); - - JSONArray resourcesJsonArray = new JSONArray(); - while(resourceIter.hasNext()){ - Resource resource = resourceIter.next(); - resourcesJsonArray.put(resource.getJson()); - } - - if(resourcesJsonArray.length() > 0){ - this.jsonObject.put(JSON_KEY_RESOURCES, resourcesJsonArray); - } - - return jsonObject.toString(); - } + @JsonIgnore + private ObjectNode getJsonNode(){ + ObjectNode objectNode = (ObjectNode) JsonUtil.getInstance().createNode(this); + // update any manually set properties + this.jsonObject.properties().iterator().forEachRemaining(f->{ + // but do not overwrite properties set via the API + if (!wellKnownKeys.contains(f.getKey())) { + objectNode.set(f.getKey(), f.getValue()); + } + }); - private JSONObject getJsonObject(){ - Iterator resourceIter = resources.iterator(); + Iterator resourceIter = resources.iterator(); - JSONArray resourcesJsonArray = new JSONArray(); + ArrayNode resourcesJsonArray = JsonUtil.getInstance().createArrayNode(); while(resourceIter.hasNext()){ Resource resource = resourceIter.next(); - resourcesJsonArray.put(resource.getJson()); + // this is ugly. If we encounter a DataResource which should be written to a file via + // manual setting, do some trickery to not write the DataResource, but a curated version + // to the package descriptor. + ObjectMapper mapper = JsonUtil.getInstance().getMapper(); + ObjectNode obj = mapper.convertValue(resource, ObjectNode.class); + if ((resource instanceof AbstractDataResource) && (resource.shouldSerializeToFile())) { + Set datafileNames = resource.getDatafileNamesForWriting(); + Set outPaths = datafileNames.stream().map((r) -> r+"."+resource.getSerializationFormat()).collect(Collectors.toSet()); + if (outPaths.size() == 1) { + obj.put(JSON_KEY_PATH, outPaths.iterator().next()); + } else { + obj.set(JSON_KEY_PATH, JsonUtil.getInstance().createArrayNode(outPaths)); + } + // If a data resource should be saved to file, it should not be inlined as well + obj.remove(JSON_KEY_DATA); + obj.put(JSON_KEY_FORMAT, resource.getSerializationFormat()); + } + obj.remove("originalReferences"); + resourcesJsonArray.add(obj); } - if(resourcesJsonArray.length() > 0){ - this.jsonObject.put(JSON_KEY_RESOURCES, resourcesJsonArray); + if(resourcesJsonArray.size() > 0){ + objectNode.set(JSON_KEY_RESOURCES, resourcesJsonArray); } - return jsonObject; + return objectNode; } - - List getErrors(){ + + /** + * Returns the validation errors of this Data Package. Always an empty List if strict mode is enabled because + * reading an invalid Package would throw an exception. + * @return List of Exceptions caught reading the Package + */ + List getErrors(){ return this.errors; } - - private void setJson(JSONObject jsonObjectSource) throws Exception { - this.jsonObject = jsonObjectSource; - + + + private void setJson(ObjectNode jsonNodeSource) throws Exception { + this.jsonObject = jsonNodeSource; + // Create Resource list, if there are resources. - if(jsonObjectSource.has(JSON_KEY_RESOURCES)){ - JSONArray resourcesJsonArray = jsonObjectSource.getJSONArray(JSON_KEY_RESOURCES); - for(int i=0; i < resourcesJsonArray.length(); i++){ - JSONObject resourceJson = resourcesJsonArray.getJSONObject(i); + if(jsonNodeSource.has(JSON_KEY_RESOURCES) && jsonNodeSource.get(JSON_KEY_RESOURCES).isArray()){ + ArrayNode resourcesJsonArray = (ArrayNode) jsonNodeSource.get(JSON_KEY_RESOURCES); + for(int i=0; i < resourcesJsonArray.size(); i++){ + ObjectNode resourceJson = (ObjectNode) resourcesJsonArray.get(i); Resource resource = null; try { - resource = Resource.build(resourceJson, basePath, isArchivePackage); + resource = Resource.fromJSON(resourceJson, basePath, isArchivePackage); } catch (DataPackageException dpe) { if(this.strictValidation){ this.jsonObject = null; this.resources.clear(); - throw dpe; - }else{ - this.errors.add(dpe); + if (dpe instanceof DataPackageValidationException) + this.errors.add((DataPackageValidationException)dpe); + else + this.errors.add(new DataPackageValidationException(dpe)); } } if(resource != null){ - addResource(resource); + addResource(resource, false); } } } else { - DataPackageException dpe = new DataPackageException("Trying to create a DataPackage from JSON, " + + DataPackageValidationException dpe = new DataPackageValidationException("Trying to create a DataPackage from JSON, " + "but no resource entries found"); if(this.strictValidation){ this.jsonObject = null; @@ -559,66 +807,39 @@ private void setJson(JSONObject jsonObjectSource) throws Exception { this.errors.add(dpe); } } - Schema schema = buildSchema (jsonObjectSource, basePath, isArchivePackage); - setFromJson(jsonObjectSource, this, schema); - - this.setName(jsonObjectSource.has(Package.JSON_KEY_ID) - ? jsonObjectSource.getString(Package.JSON_KEY_ID) - : null); - this.setVersion(jsonObjectSource.has(Package.JSON_KEY_VERSION) - ? jsonObjectSource.getString(Package.JSON_KEY_VERSION) - : null); - this.setHomepage(jsonObjectSource.has(Package.JSON_KEY_HOMEPAGE) - ? new URL(jsonObjectSource.getString(Package.JSON_KEY_HOMEPAGE)) - : null); - this.setImage(jsonObjectSource.has(Package.JSON_KEY_IMAGE) - ? jsonObjectSource.getString(Package.JSON_KEY_IMAGE) - : null); - this.setCreated(jsonObjectSource.has(Package.JSON_KEY_CREATED) - ? jsonObjectSource.getString(Package.JSON_KEY_CREATED) - : null); - this.setContributors(jsonObjectSource.has(Package.JSON_KEY_CONTRIBUTORS) - ? Contributor.fromJson(jsonObjectSource.getString(Package.JSON_KEY_CONTRIBUTORS)) - : null); - if (jsonObjectSource.has(Package.JSON_KEY_KEYWORDS)) { - JSONArray arr = jsonObject.getJSONArray(Package.JSON_KEY_KEYWORDS); - for (int i = 0; i < arr.length(); i++) { - this.addKeyword(arr.getString(i)); + setFromJson(jsonNodeSource, this); + this.setId(textValueOrNull(jsonNodeSource, Package.JSON_KEY_ID)); + this.setName(textValueOrNull(jsonNodeSource, Package.JSON_KEY_NAME)); + this.setVersion(textValueOrNull(jsonNodeSource, Package.JSON_KEY_VERSION)); + + if (jsonNodeSource.has(Package.JSON_KEY_HOMEPAGE) && + StringUtils.isNotEmpty(jsonNodeSource.get(Package.JSON_KEY_HOMEPAGE).asString())) { + this.setHomepage( new URL(jsonNodeSource.get(Package.JSON_KEY_HOMEPAGE).asString())); + } + + this.setImagePath(textValueOrNull(jsonNodeSource, Package.JSON_KEY_IMAGE)); + this.setCreated(textValueOrNull(jsonNodeSource, Package.JSON_KEY_CREATED)); + if (jsonNodeSource.has(Package.JSON_KEY_CONTRIBUTORS) && + !jsonNodeSource.get(Package.JSON_KEY_CONTRIBUTORS).isEmpty()) { + setContributors(Contributor.fromJson(jsonNodeSource.get(Package.JSON_KEY_CONTRIBUTORS))); + } + if (jsonNodeSource.has(Package.JSON_KEY_KEYWORDS)) { + ArrayNode arr = (ArrayNode) jsonObject.get(Package.JSON_KEY_KEYWORDS); + for (int i = 0; i < arr.size(); i++) { + this.addKeyword(arr.get(i).asString()); } } - List wellKnownKeys = Arrays.asList(JSON_KEY_RESOURCES, JSON_KEY_ID, JSON_KEY_VERSION, + List wellKnownKeys = Arrays.asList(JSON_KEY_NAME, JSON_KEY_RESOURCES, JSON_KEY_ID, JSON_KEY_VERSION, JSON_KEY_HOMEPAGE, JSON_KEY_IMAGE, JSON_KEY_CREATED, JSON_KEY_CONTRIBUTORS, JSON_KEY_KEYWORDS); - jsonObjectSource.keySet().forEach((k) -> { + jsonNodeSource.propertyNames().iterator().forEachRemaining((k) -> { if (!wellKnownKeys.contains(k)) { - Object obj = jsonObjectSource.get(k); - this.otherProperties.put(k, obj); + JsonNode obj = jsonNodeSource.get(k); + this.setProperty(k, obj); } }); - } - /** - * @return the profile - */ - @Override - public String getProfile() { - if (null == super.getProfile()) - return Profile.PROFILE_DATA_PACKAGE_DEFAULT; - return super.getProfile(); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getVersion() { - if (versionParts != null) { - return versionParts[0]+"."+versionParts[1]+"."+versionParts[2]; - } else - return version; + resources.forEach((r) -> r.validate(this)); + validate(); } /** @@ -629,30 +850,6 @@ public String getVersion() { */ private void setVersion(String version) { this.version = version; - if (StringUtils.isEmpty(version)) - return; - String[] parts = version.replaceAll("\\w", "").split("\\."); - if (parts.length == 3) { - try { - for (String part : parts) { - Integer.parseInt(part); - } - // do nothing if an exception is thrown, it's just - // a datapacke with sloppy version. - } catch (Exception ex) { } - // we have a SemVer version scheme - this.versionParts = new int[3]; - int cnt = 0; - for (String part : parts) { - int i = Integer.parseInt(part); - versionParts[cnt++] = i; - } - } - this.version = version; - } - - public URL getHomepage() { - return homepage; } private void setHomepage(URL homepage) { @@ -663,17 +860,14 @@ private void setHomepage(URL homepage) { this.homepage = homepage; } - - public String getImage() { - return image; - } - - private void setImage(String image) { + private void setImagePath(String image) { this.image = image; } - public ZonedDateTime getCreated() { - return created; + public void setImage(String fileName, byte[]data) throws IOException { + String sanitizedFileName = fileName.replaceAll("[\\s/\\\\]+", "_"); + this.image = sanitizedFileName; + this.imageData = data; } private void setCreated(ZonedDateTime created) { @@ -687,84 +881,118 @@ private void setCreated(String created) { setCreated(dt); } - public List getContributors() { - if (null == contributors) - return null; - return UnmodifiableList.decorate(contributors); - } - private void setContributors(Collection contributors) { if (null == contributors) return; this.contributors = new ArrayList<>(contributors); } - public void addContributor (Contributor contributor) { - if (null == contributor) + private void addKeyword(String keyword) { + if (null == keyword) return; - if (null == contributors) - contributors = new ArrayList<>(); - this.contributors.add(contributor); + if (null == keywords) + keywords = new LinkedHashSet<>(); + this.keywords.add(keyword); } - public void removeContributor (Contributor contributor) { - if (null == contributor) - return; - if (null == contributors) - return; - if (contributors.contains(contributor)) { - this.contributors.remove(contributor); + private void addResource(Resource resource, boolean validate) + throws IOException, ValidationException, DataPackageException{ + DataPackageException dpe = null; + if (resource.getName() == null){ + dpe = new DataPackageValidationException("Invalid Resource, it does not have a name property."); + if (validate) + validate(dpe); + } + if (resource instanceof AbstractDataResource) + addResource((AbstractDataResource) resource, validate); + else if (resource instanceof AbstractReferencebasedResource) + addResource((AbstractReferencebasedResource) resource, validate); + else { + dpe = checkDuplicates(resource); } + this.resources.add(resource); + if (validate) + validate(dpe); } - public Set getKeywords() { - if (null == keywords) - return null; - return UnmodifiableSet.decorate(keywords); + private void addResource(AbstractDataResource resource, boolean validate) + throws IOException, ValidationException, DataPackageException{ + DataPackageException dpe = null; + // If a name property isn't given... + if ((resource.getRawData() == null) || (resource).getFormat() == null) { + dpe = new DataPackageValidationException("Invalid Resource. The data and format properties cannot be null."); + } else { + dpe = checkDuplicates(resource); + } + if (validate) + validate(dpe); } - public void setKeywords(Set keywords) { - if (null == keywords) - return; - this.keywords = new LinkedHashSet<>(keywords); + private void addResource(AbstractReferencebasedResource resource, boolean validate) + throws IOException, ValidationException, DataPackageException{ + DataPackageException dpe = null; + if (resource.getPaths() == null) { + dpe = new DataPackageValidationException("Invalid Resource. The path property cannot be null."); + } else { + dpe = checkDuplicates(resource); + } + if (validate) + validate(dpe); } - private void addKeyword(String keyword) { - if (null == keyword) - return; - if (null == keywords) - keywords = new LinkedHashSet<>(); - this.keywords.add(keyword); - } + private void validate(DataPackageException dpe) throws IOException { + if (dpe != null) { + if (this.strictValidation) { + throw dpe; + } else { + if (dpe instanceof DataPackageValidationException) + errors.add((DataPackageValidationException)dpe); - public void removeKeyword (String keyword) { - if (null == keyword) - return; - if (null == keywords) - return; - if (keywords.contains(keyword)) { - this.keywords.remove(keyword); + errors.add(new DataPackageValidationException(dpe)); + } } + + this.validate(); } - public Object getOtherProperty(String key) { - return otherProperties.get(key); + private DataPackageException checkDuplicates(Resource resource) { + DataPackageException dpe = null; + // Check if there is duplication. + for (Resource value : this.resources) { + if (value.getName().equalsIgnoreCase(resource.getName())) { + dpe = new DataPackageException( + "A resource with the same name already exists."); + } + } + return dpe; } - public void removeOtherProperty(String key) { - if (null == key) - return; - if (null == otherProperties) - return; - if (otherProperties.keySet().contains(key)) { - otherProperties.remove(key); + private FileSystem getTargetFileSystem(File outputDir, boolean zipCompressed) throws IOException { + FileSystem outFs; + if (zipCompressed) { + if (outputDir.exists()) { + throw new DataPackageException("Cannot save into existing ZIP file: " + +outputDir.getAbsolutePath()); + } + Map env = new HashMap<>(); + env.put("create", "true"); + outFs = FileSystems.newFileSystem(URI.create("jar:" + outputDir.toURI()), env); + } else { + if (!(outputDir.isDirectory())) { + throw new DataPackageException("Target for save() exists and is a regular file: " + +outputDir.getName()); + } + outFs = outputDir.toPath().getFileSystem(); } + return outFs; } - public void setOtherProperties(String key, Object value) { - if (null == key) - return; - this.otherProperties.put(key, value); + private void writeDescriptor (FileSystem outFs, String parentDirName) throws IOException { + Path nf = outFs.getPath(parentDirName+File.separator+DATAPACKAGE_FILENAME); + try (Writer writer = Files.newBufferedWriter(nf, StandardCharsets.UTF_8, StandardOpenOption.CREATE)) { + ObjectNode jsonNode = this.getJsonNode(); + writer.write(jsonNode.toPrettyString()); + } } private static URL getParentUrl(URL urlSource) throws URISyntaxException, MalformedURLException { @@ -774,35 +1002,20 @@ private static URL getParentUrl(URL urlSource) throws URISyntaxException, Malfor : uri.resolve(".")).toURL(); } - - /** - * Check whether an input URL is valid according to DataPackage specs. - * - * From the specification: "URLs MUST be fully qualified. MUST be using either - * http or https scheme." - * - * https://frictionlessdata.io/specs/data-resource/#url-or-path - * @param url URL to test - * @return true if the String contains a URL starting with HTTP/HTTPS - */ - public static boolean isValidUrl(URL url) { - return isValidUrl(url.toExternalForm()); + // https://stackoverflow.com/a/47595502/2535335 + private static boolean isArchive(File f) throws IOException { + if ((null == f) || (!f.exists()) || (!f.isFile())) { + return false; + } + int fileSignature; + RandomAccessFile raf = new RandomAccessFile(f, "r"); + fileSignature = raf.readInt(); + raf.close(); + return fileSignature == 0x504B0304 || fileSignature == 0x504B0506 || fileSignature == 0x504B0708; } - /** - * Check whether an input string contains a valid URL. - * - * From the specification: "URLs MUST be fully qualified. MUST be using either - * http or https scheme." - * - * https://frictionlessdata.io/specs/data-resource/#url-or-path - * @param objString String to test - * @return true if the String contains a URL starting with HTTP/HTTPS - */ - public static boolean isValidUrl(String objString) { - String[] schemes = {"http", "https"}; - UrlValidator urlValidator = new UrlValidator(schemes); - return urlValidator.isValid(objString); + private static String textValueOrNull(JsonNode source, String fieldName) { + return source.has(fieldName) ? source.get(fieldName).asString() : null; } } diff --git a/src/main/java/io/frictionlessdata/datapackage/Source.java b/src/main/java/io/frictionlessdata/datapackage/Source.java new file mode 100644 index 0000000..8765a85 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/Source.java @@ -0,0 +1,24 @@ +package io.frictionlessdata.datapackage; + +public class Source { + + private String path; + + private String title; + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } +} diff --git a/src/main/java/io/frictionlessdata/datapackage/Validator.java b/src/main/java/io/frictionlessdata/datapackage/Validator.java index 0028e03..1b52e5b 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Validator.java +++ b/src/main/java/io/frictionlessdata/datapackage/Validator.java @@ -1,100 +1,136 @@ package io.frictionlessdata.datapackage; + +import tools.jackson.databind.JsonNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.tableschema.exception.ValidationException; +import io.frictionlessdata.tableschema.schema.FormalSchemaValidator; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.validator.routines.UrlValidator; + import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import org.apache.commons.validator.routines.UrlValidator; -import org.everit.json.schema.Schema; -import org.everit.json.schema.ValidationException; -import org.everit.json.schema.loader.SchemaLoader; -import org.json.JSONObject; -import org.json.JSONTokener; +import java.util.List; /** - * - * Validates against schema. + * Validates a package schema against the frictionlessdata table-schema.json (from the TableSchema project). */ public class Validator { /** * Validates a given JSON Object against the default profile schema. - * @param jsonObjectToValidate - * @throws IOException - * @throws DataPackageException - * @throws ValidationException + * + * @param jsonObjectToValidate JSON Object to validate + * @throws IOException If an I/O error occurs + * @throws DataPackageException If the profile id is invalid + * @throws ValidationException If the JSON Object is invalid */ - public void validate(JSONObject jsonObjectToValidate) throws IOException, DataPackageException, ValidationException{ - + public static void validate(JsonNode jsonObjectToValidate) throws IOException, DataPackageException, ValidationException { // If a profile value is provided. - if(jsonObjectToValidate.has(Package.JSON_KEY_PROFILE)){ - String profile = jsonObjectToValidate.getString(Package.JSON_KEY_PROFILE); - + List errors; + String profileId = Profile.PROFILE_DATA_PACKAGE_DEFAULT; + if (jsonObjectToValidate.has(Package.JSON_KEY_PROFILE)) { + profileId = jsonObjectToValidate.get(Package.JSON_KEY_PROFILE).asString(); + String[] schemes = {"http", "https"}; UrlValidator urlValidator = new UrlValidator(schemes); - - if (urlValidator.isValid(profile)) { - this.validate(jsonObjectToValidate, new URL(profile)); - }else{ - this.validate(jsonObjectToValidate, profile); + + if (urlValidator.isValid(profileId)) { + errors = validate(jsonObjectToValidate, new URL(profileId)); + } else { + errors = validate(jsonObjectToValidate, profileId); } - - }else{ + } else { // If no profile value is provided, use default value. - this.validate(jsonObjectToValidate, Profile.PROFILE_DATA_PACKAGE_DEFAULT); - } + errors = validate(jsonObjectToValidate, Profile.PROFILE_DATA_PACKAGE_DEFAULT); + } + + if (!errors.isEmpty()) { + throw new ValidationException(profileId, errors); + } } - + /** - * Validates a given JSON Object against the a given profile schema. + * Validates a given JSON Object against a given profile schema. + * * @param jsonObjectToValidate * @param profileId * @throws DataPackageException - * @throws ValidationException + * @throws ValidationException */ - public void validate(JSONObject jsonObjectToValidate, String profileId) throws DataPackageException, ValidationException{ - + private static List validate(JsonNode jsonObjectToValidate, String profileId) throws DataPackageException { InputStream inputStream = Validator.class.getResourceAsStream("/schemas/" + profileId + ".json"); - if(inputStream != null){ - JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); - Schema schema = SchemaLoader.load(rawSchema); - schema.validate(jsonObjectToValidate); // throws a ValidationException if this object is invalid - - }else{ + if (inputStream != null) { + FormalSchemaValidator schema = FormalSchemaValidator.fromJson(inputStream); + List errors = schema.validate(jsonObjectToValidate);// throws a ValidationException if this object is invalid + return errors; + } else { throw new DataPackageException("Invalid profile id: " + profileId); } - } - + /** - * * @param jsonObjectToValidate * @param schemaUrl * @throws IOException * @throws DataPackageException - * @throws ValidationException + * @throws ValidationException */ - public void validate(JSONObject jsonObjectToValidate, URL schemaUrl) throws IOException, DataPackageException, ValidationException{ - try{ + private static List validate(JsonNode jsonObjectToValidate, URL schemaUrl) throws IOException, DataPackageException { + try { InputStream inputStream = schemaUrl.openStream(); - JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); - Schema schema = SchemaLoader.load(rawSchema); - schema.validate(jsonObjectToValidate); // throws a ValidationException if this object is invalid - - }catch(FileNotFoundException e){ - throw new DataPackageException("Invalid profile schema URL: " + schemaUrl); - } + FormalSchemaValidator schema = FormalSchemaValidator.fromJson(inputStream); + List errors = schema.validate(jsonObjectToValidate);// throws a ValidationException if this object is invalid + return errors; + } catch (FileNotFoundException e) { + throw new DataPackageException("Invalid profile schema URL: " + schemaUrl); + } } - + /** * Validates a given JSON String against the default profile schema. + * * @param jsonStringToValidate * @throws IOException * @throws DataPackageException - * @throws ValidationException + * @throws ValidationException */ - public void validate(String jsonStringToValidate) throws IOException, DataPackageException, ValidationException{ - JSONObject jsonObject = new JSONObject(jsonStringToValidate); + public static void validate(String jsonStringToValidate) throws IOException, DataPackageException, ValidationException { + JsonNode jsonObject = JsonUtil.getInstance().createNode(jsonStringToValidate); validate(jsonObject); } + + /** + * Check whether an input URL is valid according to DataPackage specs. + * + * From the specification: "URLs MUST be fully qualified. MUST be using either + * http or https scheme." + * + * https://frictionlessdata.io/specs/data-resource/#url-or-path + * + * @param url URL to test + * @return true if the String contains a URL starting with HTTP/HTTPS + */ + public static boolean isValidUrl(URL url) { + return isValidUrl(url.toExternalForm()); + } + + /** + * Check whether an input string contains a valid URL. + * + * From the specification: "URLs MUST be fully qualified. MUST be using either + * http or https scheme." + * + * https://frictionlessdata.io/specs/data-resource/#url-or-path + * + * @param objString String to test + * @return true if the String contains a URL starting with HTTP/HTTPS + */ + public static boolean isValidUrl(String objString) { + String[] schemes = {"http", "https"}; + UrlValidator urlValidator = new UrlValidator(schemes); + + return urlValidator.isValid(objString); + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageException.java b/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageException.java index 5c2c90f..6e2744b 100644 --- a/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageException.java +++ b/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageException.java @@ -23,12 +23,21 @@ public DataPackageException(String msg) { super(msg); } + /** + * Constructs an instance of DataPackageException by wrapping a Throwable + * + * @param t the wrapped exception. + */ + public DataPackageException(String msg, Throwable t) { + super(msg, t); + } + /** * Constructs an instance of DataPackageException by wrapping a Throwable * * @param t the wrapped exception. */ public DataPackageException(Throwable t) { - super(t); + super(t.getMessage(), t); } } \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageValidationException.java b/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageValidationException.java new file mode 100644 index 0000000..4f06f66 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/exceptions/DataPackageValidationException.java @@ -0,0 +1,42 @@ +package io.frictionlessdata.datapackage.exceptions; + +/** + * + * + */ +public class DataPackageValidationException extends DataPackageException { + + /** + * Creates a new instance of DataPackageException without + * detail message. + */ + public DataPackageValidationException() { + } + + /** + * Constructs an instance of DataPackageException with the + * specified detail message. + * + * @param msg the detail message. + */ + public DataPackageValidationException(String msg) { + super(msg); + } + + /** + * Constructs an instance of DataPackageException by wrapping a Throwable + * + * @param t the wrapped exception. + */ + public DataPackageValidationException(Throwable t) { + super(t.getMessage(), t); + } + /** + * Constructs an instance of DataPackageException by wrapping a Throwable + * + * @param t the wrapped exception. + */ + public DataPackageValidationException(String msg, Throwable t) { + super(msg, t); + } +} \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java b/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java new file mode 100644 index 0000000..60917e1 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/fk/PackageForeignKey.java @@ -0,0 +1,94 @@ +package io.frictionlessdata.datapackage.fk; + +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.resource.Resource; +import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.exception.ForeignKeyException; +import io.frictionlessdata.tableschema.field.Field; +import io.frictionlessdata.tableschema.fk.ForeignKey; +import io.frictionlessdata.tableschema.fk.Reference; +import io.frictionlessdata.tableschema.schema.Schema; + +import java.util.ArrayList; +import java.util.List; + +/** + * PackageForeignKey is a wrapper around the {@link io.frictionlessdata.tableschema.fk.ForeignKey} class to validate foreign keys + * in the context of a data package. It checks if the referenced resource and fields exist + * in the data package and validates the foreign key constraints. + * + * This class exists because the specification of foreign keys in the tableschema specification + * is a bit awkward: it assumes that the target of a foreign key can be resolved to a different table, which is + * only possible on a datapackage level, yet the foreign key is defined as part of a Schema. + * + * In our implementation therefore, we have a ForeignKey class in the TableSchema package which only can + * validate self-referencing FKs. This class is used to resolve FKs to different resources of a package + */ +public class PackageForeignKey { + + private ForeignKey fk; + private Package datapackage; + private Resource resource; + + public PackageForeignKey(ForeignKey fk, Resource res, Package pkg) { + this.datapackage = pkg; + this.resource = res; + this.fk = fk; + } + + /** + * Formal validation of the foreign key. This method checks if the referenced resource and fields exist. + * It does not check the actual data in the tables. + * + * Verification of table data against the foreign key constraints is done in + * {@link io.frictionlessdata.datapackage.resource.AbstractResource#checkRelations}. + * + * @throws Exception if the foreign key relation is invalid. + */ + public void validate() throws Exception { + Reference reference = fk.getReference(); + // self-reference, this can be validated by the Tableschema {@link io.frictionlessdata.tableschema.fk.ForeignKey} class + if (reference.getResource().equals("")) { + for (Table table : resource.getTables()) { + fk.validate(table); + } + } else { + // validate the foreign key + Resource refResource = datapackage.getResource(reference.getResource()); + if (refResource == null) { + throw new ForeignKeyException("Reference resource not found: " + reference.getResource()); + } + List fieldNames = new ArrayList<>(); + List foreignFieldNames = new ArrayList<>(); + List lFields = fk.getFieldNames(); + Schema foreignSchema = refResource.getSchema(); + if (null == foreignSchema) { + foreignSchema = refResource.inferSchema(); + } + for (int i = 0; i < lFields.size(); i++) { + fieldNames.add(lFields.get(i)); + String foreignFieldName = reference.getFieldNames().get(i); + foreignFieldNames.add(foreignFieldName); + Field foreignField = foreignSchema.getField(foreignFieldName); + if (null == foreignField) { + throw new ForeignKeyException("Foreign key ["+fieldNames.get(i)+ "-> " + +reference.getFieldNames().get(i)+"] violation : expected: " + +reference.getFieldNames().get(i) + ", but not found"); + } + } + + } + } + + public ForeignKey getForeignKey() { + return fk; + } + + public Package getDatapackage() { + return datapackage; + } + + public Resource getResource() { + return resource; + } +} diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java index e34cfe9..f1f6f07 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java @@ -1,45 +1,55 @@ package io.frictionlessdata.datapackage.resource; -import io.frictionlessdata.datapackage.Dialect; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; -import java.nio.file.Path; +import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** - * Abstract base class for all Resources that are based on directly set data, that is not on + * Abstract base class for all Resources that are based on directly set tabular data, that is not on * data specified as files or URLs. * * @param the data format, either CSV or JSON array */ -public abstract class AbstractDataResource extends AbstractResource { - private T data; +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public abstract class AbstractDataResource extends AbstractResource { + @JsonIgnore + T data; - public AbstractDataResource(String name, T data) { + AbstractDataResource(String name, T data) { super(name); this.data = data; super.format = Resource.FORMAT_JSON; + serializeToFile = false; if (data == null) throw new DataPackageException("Invalid Resource. The data property cannot be null for a Data-based Resource."); } - /** - * @return the data - */ - public T getData() { + + @Override + @JsonProperty(JSON_KEY_DATA) + public Object getRawData() throws IOException { return data; } /** * @param data the data to set */ - public void setData(T data) { + public void setDataPoperty(T data) { this.data = data; } @Override + @JsonIgnore List readData () throws Exception{ List
tables = new ArrayList<>(); if (data != null){ @@ -57,15 +67,38 @@ List
readData () throws Exception{ } @Override - public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception { - Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; - String fileName = super.getName() + @JsonIgnore + public Set getDatafileNamesForWriting() { + String name = super.getName() .toLowerCase() .replaceAll("\\W", "_"); - List
tables = getTables(); - Path p = outputDir.resolve(fileName); - writeTableAsCsv(tables.get(0), lDialect, p); + Set names = new HashSet<>(); + names.add(JSON_KEY_DATA+"/"+name); + return names; + } + + @Override + public void validate(Package pkg) throws DataPackageValidationException { + super.validate(pkg); + try { + if (getRawData() == null) { + throw new DataPackageValidationException("Data resource must have data"); + } + + if (getFormat() == null) { + throw new DataPackageValidationException("Data resource must specify a format"); + } + } catch (Exception ex) { + if (ex instanceof DataPackageValidationException) { + errors.add((DataPackageValidationException) ex); + } + else { + errors.add(new DataPackageValidationException(ex)); + } + } } + + @JsonIgnore abstract String getResourceFormat(); } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java index b019f55..96d74bb 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java @@ -1,13 +1,26 @@ package io.frictionlessdata.datapackage.resource; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.databind.JsonNode; +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; -import org.json.JSONArray; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; -public abstract class AbstractReferencebasedResource extends AbstractResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public abstract class AbstractReferencebasedResource extends AbstractResource { Collection paths; AbstractReferencebasedResource(String name, Collection paths) { @@ -15,6 +28,7 @@ public abstract class AbstractReferencebasedResource extends AbstractResour this.paths = paths; } + @JsonIgnore public Collection getReferencesAsStrings() { final Collection strings = new ArrayList<>(); if (null != paths) { @@ -23,30 +37,140 @@ public Collection getReferencesAsStrings() { return strings; } + @Override + @JsonIgnore + public Object getRawData() throws IOException { + // If the path(s) of data file/URLs has been set. + if (paths != null){ + Iterator iter = paths.iterator(); + if (paths.size() == 1) { + return getRawData(iter.next()); + } else { + // this is probably not very useful, but it's the spec... + byte[][] retVal = new byte[paths.size()][]; + for (int i = 0; i < paths.size(); i++) { + byte[] bytes = getRawData(iter.next()); + retVal[i] = bytes; + } + return retVal; + } + } + return null; + } + /* if more than one path in our paths object, return a JSON array, else just that one object. */ - Object getPathJson() { + @JsonProperty(JSON_KEY_PATH) + JsonNode getPathJson() { List path = new ArrayList<>(getReferencesAsStrings()); if (path.size() == 1) { - return path.get(0); + return JsonUtil.getInstance().createStringNode(path.get(0)); } else { - JSONArray pathNames = new JSONArray(); - for (Object obj : path) { - pathNames.put(obj); - - } - return pathNames; + return JsonUtil.getInstance().createArrayNode(path); } } + @JsonIgnore public Collection getPaths() { return paths; } + @Override + @JsonIgnore + public Set getDatafileNamesForWriting() { + List paths = new ArrayList<>(this.getReferencesAsStrings()); + return paths.stream().map((p) -> { + if (p.toLowerCase().endsWith("."+ TableDataSource.Format.FORMAT_CSV.getLabel())){ + int i = p.toLowerCase().indexOf("."+TableDataSource.Format.FORMAT_CSV.getLabel()); + return p.substring(0, i); + } else if (p.toLowerCase().endsWith("."+TableDataSource.Format.FORMAT_JSON.getLabel())){ + int i = p.toLowerCase().indexOf("."+TableDataSource.Format.FORMAT_JSON.getLabel()); + return p.substring(0, i); + } else { + int i = p.lastIndexOf("."); + return p.substring(0, i); + } + }).collect(Collectors.toSet()); + } + + @Override + public void validate(Package pkg) throws DataPackageValidationException { + super.validate(pkg); + List paths = new ArrayList<>(getPaths()); + try { + if (paths.isEmpty()) { + throw new DataPackageValidationException("File- or URL-based resource must have at least one path"); + } + + for (T path : paths) { + String name = null; + if (path instanceof File) { + name = ((File) path).getName(); + } else if (path instanceof URL) { + name = ((URL) path).getPath(); + } + if (name == null || name.trim().isEmpty()) { + throw new DataPackageValidationException("Resource path cannot be null or empty"); + } + } + } catch (Exception ex) { + if (ex instanceof DataPackageValidationException) { + errors.add((DataPackageValidationException) ex); + } + else { + errors.add(new DataPackageValidationException(ex)); + } + } + } + + + static String sniffFormat(Collection paths) { + Set foundFormats = new HashSet<>(); + for (Object p : paths) { + String name; + if (p instanceof String) { + name = (String) p; + } else if (p instanceof File) { + name = ((File) p).getName(); + } else if (p instanceof URL) { + name = ((URL) p).getPath(); + } else { + throw new DataPackageException("Unsupported path type: " + p.getClass().getName()); + } + if (name.toLowerCase().endsWith(TableDataSource.Format.FORMAT_CSV.getLabel())) { + foundFormats.add(TableDataSource.Format.FORMAT_CSV.getLabel()); + } else if (name.toLowerCase().endsWith(TableDataSource.Format.FORMAT_JSON.getLabel())) { + foundFormats.add(TableDataSource.Format.FORMAT_JSON.getLabel()); + } else { + // something else -> not a tabular resource + int pos = name.lastIndexOf('.'); + return name.substring(pos + 1).toLowerCase(); + } + } + if (foundFormats.size() > 1) { + throw new DataPackageException("Resources cannot be mixed JSON/CSV"); + } + if (foundFormats.isEmpty()) + return TableDataSource.Format.FORMAT_CSV.getLabel(); + return foundFormats.iterator().next(); + } + abstract Table createTable(T reference) throws Exception; abstract String getStringRepresentation(T reference); + abstract byte[] getRawData(T input) throws IOException; + byte[] getRawData(InputStream inputStream) throws IOException { + byte[] retVal = null; + try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (int b; (b = inputStream.read()) != -1; ) { + out.write(b); + } + retVal = out.toByteArray(); + } + return retVal; + } + } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java index c78a352..88e8bdf 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java @@ -1,20 +1,42 @@ package io.frictionlessdata.datapackage.resource; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.core.JacksonException; +import tools.jackson.databind.ObjectMapper; import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.JSONBase; +import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.Profile; -import io.frictionlessdata.tableschema.iterator.BeanIterator; -import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.datapackage.fk.PackageForeignKey; import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.exception.ForeignKeyException; +import io.frictionlessdata.tableschema.exception.JsonSerializingException; +import io.frictionlessdata.tableschema.exception.TableIOException; +import io.frictionlessdata.tableschema.exception.TypeInferringException; +import io.frictionlessdata.tableschema.field.Field; +import io.frictionlessdata.tableschema.fk.ForeignKey; +import io.frictionlessdata.tableschema.io.FileReference; +import io.frictionlessdata.tableschema.io.URLFileReference; +import io.frictionlessdata.tableschema.iterator.BeanIterator; import io.frictionlessdata.tableschema.iterator.TableIterator; -import org.apache.commons.collections.iterators.IteratorChain; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; +import io.frictionlessdata.tableschema.util.TableSchemaUtil; +import org.apache.commons.collections4.iterators.IteratorChain; import org.apache.commons.csv.CSVFormat; -import org.json.JSONArray; -import org.json.JSONObject; +import org.apache.commons.csv.CSVPrinter; -import java.io.Writer; -import java.lang.reflect.Type; +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -23,90 +45,308 @@ * Abstract base implementation of a Resource. * Based on specs: http://frictionlessdata.io/specs/data-resource/ */ -public abstract class AbstractResource extends JSONBase implements Resource { +@JsonInclude(value = Include.NON_EMPTY, content = Include.NON_EMPTY ) +public abstract class AbstractResource extends JSONBase implements Resource { // Data properties. - private List
tables; - - // Metadata properties. - // Required properties. - private String name; - - // Recommended properties. - String profile = null; - - // Optional properties. - String title = null; - String description = null; - + @JsonIgnore + protected List
tables; + @JsonProperty("format") String format = null; - String mediaType = null; - String encoding = null; - Integer bytes = null; - String hash = null; + @JsonProperty("dialect") Dialect dialect; - JSONArray sources = null; - JSONArray licenses = null; - // Schema + @JsonProperty("schema") Schema schema = null; + @JsonIgnore + boolean serializeToFile = true; + + @JsonIgnore + String serializationFormat; + + @JsonIgnore + final List errors = new ArrayList<>(); + AbstractResource(String name){ this.name = name; - /*if (null == name) - throw new DataPackageException("Invalid Resource, it does not have a name property.");*/ + if (null == name) + throw new DataPackageException("Invalid Resource, it does not have a name property."); + } + + + @JsonProperty(JSON_KEY_SCHEMA) + public Object getSchemaForJson() { + if (originalReferences.containsKey(JSON_KEY_SCHEMA)) { + return originalReferences.get(JSON_KEY_SCHEMA); + } + if (null != schema) { + return schema; + } + return null; + } + + @JsonProperty(JSON_KEY_DIALECT) + public Object getDialectForJson() { + if (originalReferences.containsKey(JSON_KEY_DIALECT)) { + return originalReferences.get(JSON_KEY_DIALECT); + } + if (null != dialect) { + return dialect; + } + return null; } @Override public Iterator objectArrayIterator() throws Exception{ - return this.objectArrayIterator(false, false, true, false); + return this.objectArrayIterator(false, false); } @Override - public Iterator objectArrayIterator(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception{ + public Iterator objectArrayIterator(boolean extended, boolean relations) throws Exception{ ensureDataLoaded(); Iterator[] tableIteratorArray = new TableIterator[tables.size()]; int cnt = 0; for (Table table : tables) { - tableIteratorArray[cnt++] = table.iterator(keyed, extended, cast, relations); + tableIteratorArray[cnt++] = (Iterator)table.iterator(false, extended, true, relations); } return new IteratorChain(tableIteratorArray); } + public Iterator stringArrayIterator(boolean relations) throws Exception{ + ensureDataLoaded(); + Iterator[] tableIteratorArray = new TableIterator[tables.size()]; + int cnt = 0; + for (Table table : tables) { + tableIteratorArray[cnt++] = table.stringArrayIterator(relations); + } + return new IteratorChain<>(tableIteratorArray); + } + @Override public Iterator stringArrayIterator() throws Exception{ ensureDataLoaded(); Iterator[] tableIteratorArray = new TableIterator[tables.size()]; int cnt = 0; for (Table table : tables) { - tableIteratorArray[cnt++] = table.stringArrayIterator(false); + tableIteratorArray[cnt++] = table.stringArrayIterator(); + } + return new IteratorChain<>(tableIteratorArray); + } + + @Override + public Iterator> mappingIterator(boolean relations) throws Exception{ + ensureDataLoaded(); + Iterator>[] tableIteratorArray = new TableIterator[tables.size()]; + int cnt = 0; + for (Table table : tables) { + tableIteratorArray[cnt++] = table.mappingIterator(false, true, relations); } return new IteratorChain(tableIteratorArray); } + @Override + public Iterator beanIterator(Class beanType, boolean relations) throws Exception { + ensureDataLoaded(); + IteratorChain ic = new IteratorChain<>(); + for (Table table : tables) { + ic.addIterator ((Iterator) table.iterator(beanType, false)); + } + return ic; + } - public List read() throws Exception{ - return this.read(false); + /** + * Read all data from a Resource, each row as String arrays. This can be used for smaller datapackages, + * but for huge or unknown sizes, reading via iterator is preferred, as this method loads all data into RAM. + * + * It can be configured to return table rows with relations to other data sources resolved + * + * The method uses Iterators provided by {@link Table} class, and is roughly implemented after + * https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py + * + * @param relations true: follow relations + * @return A list of table rows. + * @throws Exception if parsing the data fails + * + */ + @JsonIgnore + public List getData(boolean relations) throws Exception{ + List retVal = new ArrayList<>(); + ensureDataLoaded(); + Iterator iter = stringArrayIterator(relations); + while (iter.hasNext()) { + retVal.add(iter.next()); + } + return retVal; } - public List read (boolean cast) throws Exception{ - List retVal = new ArrayList<>(); + /** + * Read all data from a Resource, each row as Map objects. This can be used for smaller datapackages, + * but for huge or unknown sizes, reading via iterator is preferred, as this method loads all data into RAM. + * + * The method returns Map<String,Object> where key is the header name, and val is the data. + * It can be configured to return table rows with relations to other data sources resolved + * + * The method uses Iterators provided by {@link Table} class, and is roughly implemented after + * https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py + * + * @param relations true: follow relations + * @return A list of table rows. + * @throws Exception if parsing the data fails + * + */ + @Override + public List> getMappedData(boolean relations) throws Exception { + List> retVal = new ArrayList<>(); ensureDataLoaded(); - Iterator iter = objectArrayIterator(false, false, cast, false); + Iterator[] tableIteratorArray = new TableIterator[tables.size()]; + int cnt = 0; + for (Table table : tables) { + tableIteratorArray[cnt++] = table.iterator(true, false, true, relations); + } + Iterator iter = new IteratorChain<>(tableIteratorArray); + while (iter.hasNext()) { + retVal.add((Map)iter.next()); + } + return retVal; + } + + /** + * Most customizable method to retrieve all data in a Resource. Parameters match those in + * {@link io.frictionlessdata.tableschema.Table#iterator(boolean, boolean, boolean, boolean)}. Data can be + * returned as: + *
    + *
  • String arrays,
    • + *
  • as Object arrays (parameter `cast` = true),
    • + *
  • as a Map<String,Object> where key is the header name, and val is the data (parameter `keyed` = true), + *
  • or in an "extended" form (parameter `extended` = true) that returns an Object array where the first entry is the + * row number, the second is a String array holding the headers, and the third is an Object array holding + * the row data.
    • + *
+ * The following rules apply: + *
    + *
  • if no Schema is present, rows will always return string, not objects, as if `cast` was always off
    • + *
  • if `extended` is true, then `cast` is also true, but `keyed` is false
    • + *
  • if `keyed` is true, then `cast` is also true, but `extended` is false
    • + *
+ * @param keyed returns data as Maps + * @param extended returns data in "extended form" + * @param cast returns data as Objects, not Strings + * @param relations resolves relations + * @return List of data objects + * @throws Exception if reading data fails + */ + public List getData(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception{ + List retVal = new ArrayList<>(); + ensureDataLoaded(); + Iterator iter; + if (keyed) { + iter = mappingIterator(relations); + } else if (cast) { + iter = objectArrayIterator(extended, relations); + } else { + iter = stringArrayIterator(relations); + } while (iter.hasNext()) { retVal.add(iter.next()); } return retVal; } + + @JsonIgnore + public String getDataAsJson() { + List> rows = new ArrayList<>(); + Schema schema = (null != this.schema) ? this.schema : this.inferSchema(); + try { + ensureDataLoaded(); + } catch (Exception e) { + throw new DataPackageException(e); + } + + for (Table table : tables) { + Iterator iter = table.iterator(false, false, true, false); + iter.forEachRemaining((rec) -> { + Object[] row = (Object[]) rec; + Map obj = new LinkedHashMap<>(); + int i = 0; + for (Field field : schema.getFields()) { + Object s = row[i]; + obj.put(field.getName(), field.formatValueForJson(s)); + i++; + } + rows.add(obj); + }); + } + + String retVal; + ObjectMapper mapper = JsonUtil.getInstance().getMapper(); + try { + retVal = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(rows); + } catch (JacksonException ex) { + throw new JsonSerializingException(ex); + } + return retVal; + } + + @JsonIgnore + public String getDataAsCsv() { + Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; + Schema schema = (null != this.schema) ? this.schema : inferSchema(); + + return getDataAsCsv(lDialect, schema); + } + + public String getDataAsCsv(Dialect dialect, Schema schema) { + StringBuilder out = new StringBuilder(); + try { + ensureDataLoaded(); + if (null == schema) { + return getDataAsCsv(dialect, inferSchema()); + } + CSVFormat locFormat = dialect.toCsvFormat(); + locFormat = locFormat.builder().setHeader(schema.getHeaders()).get(); + CSVPrinter csvPrinter = new CSVPrinter(out, locFormat); + String[] headerNames = schema.getHeaders(); + + for (Table table : tables) { + String[] headers = table.getHeaders(); + if (null == headerNames) { + headerNames = headers; + } + Map mapping = TableSchemaUtil.createSchemaHeaderMapping( + headers, + headerNames, + table.getTableDataSource().hasReliableHeaders()); + + appendCSVDataToPrinter(table, mapping, schema, csvPrinter); + } + + csvPrinter.close(); + } catch (IOException ex) { + throw new TableIOException(ex); + } catch (Exception e) { + throw new DataPackageException(e); + } + String result = out.toString(); + if (result.endsWith("\n")) { + result = result.substring(0, result.length() - 1); + } + if (result.endsWith("\r")) { + result = result.substring(0, result.length() - 1); + } + return result; + } + + @Override - public List read (Class beanClass) throws Exception { + public List getData(Class beanClass) throws Exception { List retVal = new ArrayList(); ensureDataLoaded(); for (Table t : tables) { - final BeanIterator iter = new BeanIterator(t, beanClass); + final BeanIterator iter = (BeanIterator) t.iterator(beanClass, false); while (iter.hasNext()) { retVal.add(iter.next()); } @@ -115,189 +355,297 @@ public List read (Class beanClass) throws Exception { } @Override + @JsonIgnore public String[] getHeaders() throws Exception{ ensureDataLoaded(); return tables.get(0).getHeaders(); } @Override + @JsonIgnore public List
getTables() throws Exception { ensureDataLoaded(); return tables; } - /** - * Get JSON representation of the object. - * @return a JSONObject representing the properties of this object - */ - public JSONObject getJson(){ - //FIXME: Maybe use something lke GSON so we don't have to explicitly - //code this... - JSONObject json = new JSONObject(new LinkedHashMap()); - - // Null values will not actually be "put," as per JSONObject specs. - json.put(JSON_KEY_NAME, this.getName()); + public void checkRelations(Package pkg) { + if (null != schema) { + List fks = new ArrayList<>(); + for (ForeignKey fk : schema.getForeignKeys()) { + String resourceName = fk.getReference().getResource(); + Resource referencedResource; + if (null != resourceName) { + if (resourceName.isEmpty()) { + referencedResource = this; + } else { + referencedResource = pkg.getResource(resourceName); + } + if (null == referencedResource) { + throw new DataPackageValidationException("Foreign key references non-existent referencedResource: " + resourceName); + } + try { + PackageForeignKey pFK = new PackageForeignKey(fk, this, pkg); + fks.add(pFK); + pFK.validate(); + } catch (Exception e) { + throw new DataPackageValidationException("Foreign key validation failed: " + resourceName, e); + } + } + } - if (this instanceof AbstractReferencebasedResource) { - json.put(JSON_KEY_PATH, ((AbstractReferencebasedResource)this).getPathJson()); - } - if (this instanceof AbstractDataResource) { - json.put(JSON_KEY_DATA, ((AbstractDataResource)this).getData()); + try { + Map> map = new HashMap<>(); + for (PackageForeignKey fk : fks) { + String refResourceName = fk.getForeignKey().getReference().getResource(); + Resource refResource = pkg.getResource(refResourceName); + List data = refResource.getData(true, false, true, false); + map.put(fk, data); + } + List data = this.getData(true, false, true, false); + for (Object d : data) { + Map row = (Map) d; + for (String key : row.keySet()) { + for (PackageForeignKey fk : map.keySet()) { + if (fk.getForeignKey().getFieldNames().contains(key)) { + ListrefData = map.get(fk); + Map fieldMapping = fk.getForeignKey().getFieldMapping(); + String refFieldName = fieldMapping.get(key); + Object fkVal = row.get(key); + if (fkVal == null) { + continue; + } + boolean found = false; + + for (Object refRow : refData) { + Map refRowMap = (Map) refRow; + Object refVal = refRowMap.get(refFieldName); + if (Objects.equals(fkVal, refVal)) { + found = true; + break; + } + } + if (!found) { + throw new ForeignKeyException("Foreign key validation failed: " + + fk.getForeignKey().getFieldNames() + " -> " + + fk.getForeignKey().getReference().getFieldNames() + ": '" + + fkVal + "' not found in resource '" + + fk.getForeignKey().getReference().getResource()+"'."); + } + } + } + } + + } + } catch (Exception e) { + throw new DataPackageValidationException("Error reading data with relations: " + e.getMessage(), e); + } } - json.put(JSON_KEY_PROFILE, this.profile); - json.put(JSON_KEY_TITLE, this.title); - json.put(JSON_KEY_DESCRIPTION, this.description); - json.put(JSON_KEY_FORMAT, this.format); - json.put(JSON_KEY_MEDIA_TYPE, this.getMediaType()); - json.put(JSON_KEY_ENCODING, this.getEncoding()); - json.put(JSON_KEY_BYTES, this.getBytes()); - json.put(JSON_KEY_HASH, this.getHash()); - json.put(JSON_KEY_SOURCES, this.getSources()); - json.put(JSON_KEY_LICENSES, this.getLicenses()); - - Object schemaObj = originalReferences.get(JSONBase.JSON_KEY_SCHEMA); - json.put(JSON_KEY_SCHEMA, schemaObj); - - Object dialectObj = originalReferences.get(JSONBase.JSON_KEY_DIALECT); - json.put(JSON_KEY_DIALECT, dialectObj); - return json; } - /** - * @return the name - */ - @Override - public String getName() { - return name; - } + public void validate(Package pkg) { - /** - * @param name the name to set - */ - @Override - public void setName(String name) { - this.name = name; - } + try { + // Validate required fields + if (getName() == null || getName().trim().isEmpty()) { + throw new DataPackageValidationException("Resource must have a name"); + } - /** - * @return the profile - */ - @Override - public String getProfile() { - if (null == profile) - return Profile.PROFILE_DATA_RESOURCE_DEFAULT; - return profile; - } + // Validate name format (alphanumeric, dash, underscore only) + if (!getName().matches("^[a-zA-Z0-9_-]+$")) { + throw new DataPackageValidationException("Resource name must contain only alphanumeric characters, dashes, and underscores"); + } - /** - * @param profile the profile to set - */ - @Override - public void setProfile(String profile) { - this.profile = profile; - } + // Validate profile + String profile = getProfile(); + if (profile != null && !isValidProfile(profile)) { + throw new DataPackageValidationException("Invalid resource profile: " + profile); + } - /** - * @return the title - */ - @Override - public String getTitle() { - return title; + if (null != schema) { + try { + schema.validate(); + } catch (DataPackageValidationException e) { + throw new DataPackageValidationException("Schema validation failed for resource " + getName() + ": " + e.getMessage(), e); + } + } + if (null == tables) + return; + // will validate schema against data + tables.forEach(Table::validate); + checkRelations(pkg); + } catch (Exception ex) { + if (ex instanceof DataPackageValidationException) { + errors.add((DataPackageValidationException) ex); + } + else { + errors.add(new DataPackageValidationException(ex)); + } + } } - /** - * @param title the title to set - */ - @Override - public void setTitle(String title) { - this.title = title; - } + public void writeSchema(Path parentFilePath) throws IOException { + String relPath = getPathForWritingSchema(); + if (null == originalReferences.get(JSONBase.JSON_KEY_SCHEMA) && Objects.nonNull(relPath)) { + originalReferences.put(JSONBase.JSON_KEY_SCHEMA, relPath); + } - /** - * @return the description - */ - @Override - public String getDescription() { - return description; + if (null != relPath) { + boolean isRemote; + try { + // don't try to write schema files that came from remote, let's just add the URL to the descriptor + URI uri = new URI(relPath); + isRemote = (null != uri.getScheme()) && + (uri.getScheme().equals("http") || uri.getScheme().equals("https")); + if (isRemote) + return; + } catch (URISyntaxException ignored) {} + Path p; + if (parentFilePath.toString().isEmpty()) { + p = parentFilePath.getFileSystem().getPath(relPath); + } else { + p = parentFilePath.resolve(relPath); + } + writeSchema(p, schema); + } } - /** - * @param description the description to set - */ - @Override - public void setDescription(String description) { - this.description = description; + private static void writeSchema(Path parentFilePath, Schema schema) throws IOException { + if (!Files.exists(parentFilePath)) { + Files.createDirectories(parentFilePath); + } + Files.deleteIfExists(parentFilePath); + try (Writer wr = Files.newBufferedWriter(parentFilePath, StandardCharsets.UTF_8)) { + wr.write(schema.asJson()); + } } + public void writeDialect(Path parentFilePath) throws IOException { + if (null == dialect) + return; + String relPath = getPathForWritingDialect(); + if (null == originalReferences.get(JSONBase.JSON_KEY_DIALECT) && Objects.nonNull(relPath)) { + originalReferences.put(JSONBase.JSON_KEY_DIALECT, relPath); + } - /** - * @return the mediaType - */ - @Override - public String getMediaType() { - return mediaType; - } - - /** - * @param mediaType the mediaType to set - */ - @Override - public void setMediaType(String mediaType) { - this.mediaType = mediaType; + if (null != relPath) { + boolean isRemote; + try { + // don't try to write schema files that came from remote, let's just add the URL to the descriptor + URI uri = new URI(relPath); + isRemote = (null != uri.getScheme()) && + (uri.getScheme().equals("http") || uri.getScheme().equals("https")); + if (isRemote) + return; + } catch (URISyntaxException ignored) {} + Path p; + if (parentFilePath.toString().isEmpty()) { + p = parentFilePath.getFileSystem().getPath(relPath); + } else { + p = parentFilePath.resolve(relPath); + } + writeDialect(p, dialect); + } } - /** - * @return the encoding - */ - @Override - public String getEncoding() { - return encoding; + private static void writeDialect(Path parentFilePath, Dialect dialect) throws IOException { + if (!Files.exists(parentFilePath)) { + Files.createDirectories(parentFilePath); + } + Files.deleteIfExists(parentFilePath); + try (Writer wr = Files.newBufferedWriter(parentFilePath, StandardCharsets.UTF_8)) { + wr.write(dialect.asJson()); + } } - /** - * @param encoding the encoding to set - */ @Override - public void setEncoding(String encoding) { - this.encoding = encoding; + public Schema inferSchema() throws TypeInferringException { + Schema schema; + try { + List
tables = getTables(); + String[] headers = getHeaders(); + schema = tables.get(0).inferSchema(headers, -1); + for (int i = 1; i < tables.size(); i++) { + Schema schema2 = tables.get(i).inferSchema(); + for (Field field : schema2.getFields()) { + if (null == schema.getField(field.getName())) { + throw new TypeInferringException("Found field mismatch in Tables of Resource: " + getName()); + } + } + } + } catch (Exception e) { + throw new DataPackageException("Error inferring schema", e); + } + return schema; } /** - * @return the bytes + * Construct a path to write out the Schema for this Resource + * @return a String containing a relative path for writing or null */ @Override - public Integer getBytes() { - return bytes; + @JsonIgnore + public String getPathForWritingSchema() { + Schema resSchema = getSchema(); + // write out schema file only if not null or URL + FileReference ref = (null != resSchema) ? resSchema.getReference() : null; + return getPathForWritingSchemaOrDialect(JSON_KEY_SCHEMA, resSchema, ref); } /** - * @param bytes the bytes to set + * Construct a path to write out the Dialect for this Resource + * @return a String containing a relative path for writing or null */ @Override - public void setBytes(Integer bytes) { - this.bytes = bytes; + @JsonIgnore + public String getPathForWritingDialect() { + Dialect dialect = getDialect(); + // write out dialect file only if not null or URL + FileReference ref = (null != dialect) ? dialect.getReference() : null; + return getPathForWritingSchemaOrDialect(JSON_KEY_DIALECT, dialect, ref); } /** - * @return the hash + * If we don't have a object, return null (nothing to serialize) + * If we have a object, but it was read from an URL, return null (DataPackage will just use the URL) + * If there is a object in the first place and it is not URL based or freshly created, + * construct a relative file path for writing + * If we have a object, but it is freshly created, and the Resource data should be written to file, + * create a file name from the Resource name + * If we have a object, but it is freshly created, and the Resource data should not be written to file, + * return null + * @return a String containing a relative path for writing or null */ - @Override - public String getHash() { - return hash; + private String getPathForWritingSchemaOrDialect(String key, Object objectWithRes, FileReference reference) { + // write out schema file only if not null or URL + if (null == objectWithRes) { + return null; + } else if ((reference instanceof URLFileReference)){ + return null; + } else if (getOriginalReferences().containsKey(key)) { + return getOriginalReferences().get(key); + } else if (null != reference) { + return key + "/" + reference.getFileName(); + } else if (this.shouldSerializeToFile()) { + return key + "/" + name.toLowerCase().replaceAll("\\W", "")+".json"; + } else + return null; } /** - * @param hash the hash to set + * @return the profile */ @Override - public void setHash(String hash) { - this.hash = hash; + public String getProfile() { + if (null == profile) + return Profile.PROFILE_DATA_RESOURCE_DEFAULT; + return profile; } /** * @return the dialect */ @Override + @JsonIgnore public Dialect getDialect() { return dialect; } @@ -310,77 +658,94 @@ public void setDialect(Dialect dialect) { this.dialect = dialect; } + /** + * @param profile the profile to set + */ + @Override + public void setProfile(String profile){ + if (null != profile) { + if ((profile.equals(Profile.PROFILE_TABULAR_DATA_PACKAGE)) + || (profile.equals(Profile.PROFILE_DATA_PACKAGE_DEFAULT))) { + throw new DataPackageValidationException("Cannot set profile " + profile + " on a resource"); + } + } + this.profile = profile; + } + @Override + @JsonProperty(JSON_KEY_FORMAT) public String getFormat() { return format; } @Override + @JsonProperty(JSON_KEY_FORMAT) public void setFormat(String format) { this.format = format; } @Override + @JsonIgnore public Schema getSchema(){ return this.schema; } @Override + @JsonIgnore public void setSchema(Schema schema) { this.schema = schema; } - @Override - public String getSchemaReference(){ - return super.getSchemaReference(); - } - + @JsonIgnore public String getDialectReference() { if (null == originalReferences.get(JSONBase.JSON_KEY_DIALECT)) return null; return originalReferences.get(JSONBase.JSON_KEY_DIALECT).toString(); } + @JsonIgnore CSVFormat getCsvFormat() { Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; return lDialect.toCsvFormat(); } - /** - * @return the sources - */ @Override - public JSONArray getSources() { - return sources; + @JsonIgnore + public boolean shouldSerializeToFile() { + return serializeToFile; } - /** - * @param sources the sources to set - */ @Override - public void setSources(JSONArray sources) { - this.sources = sources; + public void setShouldSerializeToFile(boolean serializeToFile) { + this.serializeToFile = serializeToFile; } - /** - * @return the licenses - */ @Override - public JSONArray getLicenses() { - return licenses; + public void setSerializationFormat(String format) { + if ((null == format) || (format.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) + || format.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) { + this.serializationFormat = format; + } else + throw new DataPackageException("Serialization format "+format+" is unknown"); } /** - * @param licenses the licenses to set + * if an explicit serialisation format was set, return this. Alternatively return the default + * {@link io.frictionlessdata.tableschema.tabledatasource.TableDataSource.Format} as a String + * @return Serialisation format, either "csv" or "json" */ - @Override - public void setLicenses(JSONArray licenses) { - this.licenses = licenses; + @JsonIgnore + public String getSerializationFormat() { + if (null != serializationFormat) + return serializationFormat; + return format; } abstract List
readData() throws Exception; - private List
ensureDataLoaded () throws Exception { + public abstract Set getDatafileNamesForWriting(); + + List
ensureDataLoaded () throws Exception { if (null == tables) { tables = readData(); } @@ -388,7 +753,78 @@ private List
ensureDataLoaded () throws Exception { } @Override - public void writeTableAsCsv(Table t, Dialect dialect, Path outputFile) throws Exception { + public void writeData(Writer out) throws Exception { + Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; + List
tables = getTables(); + for (Table t : tables) { + if (serializationFormat.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) { + t.writeCsv(out, lDialect.toCsvFormat()); + } else if (serializationFormat.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) { + out.write(t.asJson()); + } + } + } + + @Override + public void writeData(Path outputDir) throws Exception { + Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; + boolean isNonTabular = ((profile != null) && (profile.equals(Profile.PROFILE_DATA_RESOURCE_DEFAULT))); + isNonTabular = (isNonTabular | (null == serializationFormat)); + List
tables = isNonTabular ? null : getTables(); + + Set paths = getDatafileNamesForWriting(); + + int cnt = 0; + for (String fName : paths) { + String fileName = fName+"."+getSerializationFormat(); + Path p; + FileSystem fileSystem = outputDir.getFileSystem(); + if (outputDir.toString().isEmpty()) { + p = fileSystem.getPath(fileName); + } else { + p = outputDir.resolve(fileName); + } + if (!Files.exists(p)) { + Files.createDirectories(p); + } + Files.deleteIfExists(p); + + + if (isNonTabular) { + byte [] data = (byte[])this.getRawData(); + try (OutputStream out = Files.newOutputStream(p)) { + out.write(data); + } + } else { + Table t = tables.get(cnt++); + try (Writer wr = Files.newBufferedWriter(p, StandardCharsets.UTF_8)) { + if (serializationFormat.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) { + t.writeCsv(wr, lDialect.toCsvFormat()); + } else if (serializationFormat.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) { + wr.write(t.asJson()); + } + } + } + } + } + + + private static boolean isValidProfile(String profile) { + return profile.equals(Profile.PROFILE_DATA_RESOURCE_DEFAULT) || + profile.equals(Profile.PROFILE_TABULAR_DATA_RESOURCE) || + profile.startsWith("http://") || + profile.startsWith("https://"); + } + + + /** + * Write the Table as CSV into a file inside `outputDir`. + * + * @param outputFile the file to write to. + * @param dialect the CSV dialect to use for writing + * @throws Exception if something fails while writing + */ + void writeTableAsCsv(Table t, Dialect dialect, Path outputFile) throws Exception { if (!Files.exists(outputFile)) { Files.createDirectories(outputFile); } @@ -397,4 +833,34 @@ public void writeTableAsCsv(Table t, Dialect dialect, Path outputFile) throws Ex t.writeCsv(wr, dialect.toCsvFormat()); } } + + /** + * Append the data to a {@link org.apache.commons.csv.CSVPrinter}. Column sorting is according to the mapping + * @param mapping the mapping of the column numbers in the CSV file to the column numbers in the data source + * @param schema the Schema to use for formatting the data + * @param csvPrinter the CSVPrinter to write to + */ + private void appendCSVDataToPrinter(Table table, Map mapping, Schema schema, CSVPrinter csvPrinter) { + Iterator iter = table.iterator(false, false, true, false); + iter.forEachRemaining((rec) -> { + Object[] row = (Object[])rec; + Object[] sortedRec = new Object[row.length]; + for (int i = 0; i < row.length; i++) { + sortedRec[mapping.get(i)] = row[i]; + } + List obj = new ArrayList<>(); + int i = 0; + for (Field field : schema.getFields()) { + Object s = sortedRec[i]; + obj.add(field.formatValueAsString(s)); + i++; + } + + try { + csvPrinter.printRecord(obj); + } catch (Exception ex) { + throw new TableIOException(ex); + } + }); + } } \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java index 2ab82e8..bb3ff5e 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java @@ -1,6 +1,10 @@ package io.frictionlessdata.datapackage.resource; -public class CSVDataResource extends AbstractDataResource { +import com.fasterxml.jackson.annotation.JsonInclude; +import io.frictionlessdata.datapackage.Profile; + +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class CSVDataResource extends AbstractDataResource { public CSVDataResource(String name, String data) { super(name, data); @@ -11,4 +15,9 @@ public CSVDataResource(String name, String data) { String getResourceFormat() { return Resource.FORMAT_CSV; } + + @Override + public String getProfile() { + return Profile.PROFILE_TABULAR_DATA_RESOURCE; + } } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java index 661ecb1..1c0c646 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java @@ -1,12 +1,18 @@ package io.frictionlessdata.datapackage.resource; -import io.frictionlessdata.datapackage.Dialect; -import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; -import io.frictionlessdata.tableschema.datasourceformats.DataSourceFormat; -import org.apache.commons.csv.CSVFormat; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -14,16 +20,36 @@ import java.util.List; -public class FilebasedResource extends AbstractReferencebasedResource { +@JsonInclude(value = Include.NON_EMPTY, content = Include.NON_EMPTY ) +public class FilebasedResource extends AbstractReferencebasedResource { + @JsonIgnore private File basePath; + + @JsonIgnore private boolean isInArchive; - public FilebasedResource(String name, Collection paths, File basePath) { + + /** + * The charset (encoding) for writing + */ + @JsonIgnore + private final Charset charset = StandardCharsets.UTF_8; + + public FilebasedResource(String name, Collection paths, File basePath, Charset encoding) { super(name, paths); + this.encoding = encoding.name(); if (null == paths) { - throw new DataPackageException("Invalid Resource. " + + throw new DataPackageValidationException("Invalid Resource. " + "The path property cannot be null for file-based Resources."); } + String format = sniffFormat(paths); + if (format.equals(TableDataSource.Format.FORMAT_JSON.getLabel()) + || format.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) { + this.setSerializationFormat(format); + } else { + super.setFormat(format); + } + this.basePath = basePath; for (File path : paths) { /* from the spec: "SECURITY: / (absolute path) and ../ (relative parent path) @@ -33,17 +59,55 @@ public FilebasedResource(String name, Collection paths, File basePath) { https://frictionlessdata.io/specs/data-resource/index.html#url-or-path */ if (path.isAbsolute()) { - throw new DataPackageException("Path entries for file-based Resources cannot be absolute"); + throw new DataPackageValidationException("Path entries for file-based Resources cannot be absolute"); } } + serializeToFile = true; } + public FilebasedResource(String name, Collection paths, File basePath) { + this(name, paths, basePath, Charset.defaultCharset()); + } + + + @JsonIgnore + public String getSerializationFormat() { + if (null != serializationFormat) + return serializationFormat; + if (null != format) { + return format; + } + return sniffFormat(paths); + } + + public static FilebasedResource fromSource(String name, Collection paths, File basePath, Charset encoding) { + FilebasedResource r = new FilebasedResource(name, paths, basePath); + r.encoding = encoding.name(); + return r; + } + + @JsonIgnore public File getBasePath() { return basePath; } @Override - Table createTable(File reference) throws Exception { + @JsonIgnore + byte[] getRawData(File input) throws IOException { + if (this.isInArchive) { + String fileName = input.getPath().replaceAll("\\\\", "/"); + return getZipFileContentAsString (basePath.toPath(), fileName, true).getBytes(charset); + } else { + File file = new File(this.basePath, input.getPath()); + try (InputStream inputStream = Files.newInputStream(file.toPath())) { + return getRawData(inputStream); + } + } + + } + + @Override + Table createTable(File reference) { return Table.fromSource(reference, basePath, schema, getCsvFormat()); } @@ -55,27 +119,38 @@ String getStringRepresentation(File reference) { } @Override + @JsonIgnore + public String[] getHeaders() throws Exception{ + if ((null != profile) && (profile.equals(Profile.PROFILE_DATA_PACKAGE_DEFAULT))) { + return null; + } + ensureDataLoaded(); + return tables.get(0).getHeaders(); + } + + @Override + @JsonIgnore List
readData () throws Exception{ - List
tables = new ArrayList<>(); + List
tables; if (this.isInArchive) { - tables = readfromZipFile(); + tables = readFromZipFile(); } else { - tables = readfromOrdinaryFile(); + tables = readFromOrdinaryFile(); } return tables; } - private List
readfromZipFile() throws Exception { + private List
readFromZipFile() throws IOException { List
tables = new ArrayList<>(); for (File file : paths) { String fileName = file.getPath().replaceAll("\\\\", "/"); - String content = getZipFileContentAsString (basePath.toPath(), fileName); + String content = getZipFileContentAsString (basePath.toPath(), fileName, true); Table table = Table.fromSource(content, schema, getCsvFormat()); tables.add(table); } return tables; } - private List
readfromOrdinaryFile() throws Exception { + private List
readFromOrdinaryFile() throws IOException { List
tables = new ArrayList<>(); for (File file : paths) { /* from the spec: "SECURITY: / (absolute path) and ../ (relative parent path) @@ -92,32 +167,7 @@ private List
readfromOrdinaryFile() throws Exception { return tables; } - @Override - public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception { - Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; - List paths = new ArrayList<>(getReferencesAsStrings()); - int cnt = 0; - for (String fileName : paths) { - List
tables = getTables(); - Table t = tables.get(cnt++); - Path p; - if (outputDir.toString().isEmpty()) { - p = outputDir.getFileSystem().getPath(fileName); - if (!Files.exists(p)) { - Files.createDirectories(p); - } - } else { - if (!Files.exists(outputDir)) { - Files.createDirectories(outputDir); - } - p = outputDir.resolve(fileName); - } - - Files.deleteIfExists(p); - writeTableAsCsv(t, lDialect, p); - } - } - + @JsonIgnore public void setIsInArchive(boolean isInArchive) { this.isInArchive = isInArchive; } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java index d5fac9e..6e4125e 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java @@ -1,16 +1,30 @@ package io.frictionlessdata.datapackage.resource; -import org.json.JSONArray; +import com.fasterxml.jackson.annotation.JsonInclude; +import tools.jackson.databind.node.ArrayNode; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; -public class JSONDataResource extends AbstractDataResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class JSONDataResource extends AbstractDataResource { public JSONDataResource(String name, String json) { - super(name, new JSONArray(json)); + this(name, JsonUtil.getInstance().createArrayNode(json)); + } + + public JSONDataResource(String name, ArrayNode json) { + super(name, json); super.format = getResourceFormat(); } @Override String getResourceFormat() { - return Resource.FORMAT_JSON; + return TableDataSource.Format.FORMAT_JSON.getLabel(); + } + + @Override + public String getProfile() { + return Profile.PROFILE_TABULAR_DATA_RESOURCE; } } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/JSONObjectResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/JSONObjectResource.java new file mode 100644 index 0000000..00ce4bb --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/resource/JSONObjectResource.java @@ -0,0 +1,34 @@ +package io.frictionlessdata.datapackage.resource; + +import com.fasterxml.jackson.annotation.JsonInclude; +import tools.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; + +/** + * A non-tabular resource that holds JSON object data + */ +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class JSONObjectResource extends AbstractDataResource { + + public JSONObjectResource(String name, String json) { + super(name, (ObjectNode)JsonUtil.getInstance().createNode(json)); + super.format = getResourceFormat(); + } + + public JSONObjectResource(String name, ObjectNode json) { + super(name, json); + super.format = getResourceFormat(); + } + + @Override + String getResourceFormat() { + return TableDataSource.Format.FORMAT_JSON.getLabel(); + } + + @Override + public String getProfile() { + return Profile.PROFILE_DATA_RESOURCE_DEFAULT; + } +} diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java index 6783ef1..b0e29d0 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java @@ -1,170 +1,322 @@ package io.frictionlessdata.datapackage.resource; -import io.frictionlessdata.datapackage.Dialect; -import io.frictionlessdata.datapackage.JSONBase; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; +import tools.jackson.databind.node.StringNode; +import io.frictionlessdata.datapackage.*; +import io.frictionlessdata.datapackage.Package; import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.exception.TypeInferringException; import io.frictionlessdata.tableschema.iterator.TableIterator; -import org.json.JSONArray; -import org.json.JSONObject; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.lang3.StringUtils; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.Writer; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.Charset; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; +import java.util.*; -import static io.frictionlessdata.datapackage.Package.isValidUrl; +import static io.frictionlessdata.datapackage.JSONBase.JSON_KEY_DATA; +import static io.frictionlessdata.datapackage.Validator.isValidUrl; /** - * Interface for a Resource. + * Interface for a Resource. The essence of a Data Resource is a locator for the data it describes. + * A range of other properties can be declared to provide a richer set of metadata. + * + * The various subclasses of Resource allow you to implement several resource types: + * + * 1. Child classes of `AbstractDataResource` - for inline data (CSV strings or JSON arrays). Will be serialized to JSON objects + * in the datapackage.json + * - `CSVDataResource` - for CSV string data + * - `JSONDataResource` - for JSON array data + * + * 2. Child classes of `AbstractReferencebasedResource` - for external data references + * - `FilebasedResource` - for local files + * - `URLbasedResource` - for remote URLs + * + * 3. Non-tabular resources can be implemented by `JSONObjectResource` and custom implementations + * * Based on specs: http://frictionlessdata.io/specs/data-resource/ */ -public interface Resource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public interface Resource extends BaseInterface { String FORMAT_CSV = "csv"; String FORMAT_JSON = "json"; - List
getTables() throws Exception ; - - JSONObject getJson(); + /** + * Create a new ResourceBuilder that allows for building Resources + * @param resourceName the name of the resource + * @return a new ResourceBuilder instance + */ + static ResourceBuilder builder(String resourceName) { + return ResourceBuilder.create(resourceName); + } - List read (boolean cast) throws Exception; - List read (Class beanClass) throws Exception; + /** + * Return the {@link Table} objects underlying the Resource. + * @return Table(s) + * @throws Exception if reading the tables fails. + */ + @JsonIgnore + List
getTables() throws Exception ; /** - * Write all the data in this resource as CSV into one or more - * files inside `outputDir`, depending on how many tables this - * Resource holds. + * Read all data from a Resource, unmapped and not transformed. This is useful for non-tabular resources + * + * @return Contents of the resource file or URL. + * @throws IOException if reading the data fails * - * @param outputDir the directory to write to. Code must create - * files as needed. - * @param dialect the CSV dialect to use for writing - * @throws Exception if something fails while writing */ - void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception; + @JsonProperty(JSON_KEY_DATA) + public Object getRawData() throws IOException; /** - * Write the Table as CSV into a file inside `outputDir`. + * Read all data from a Resource, each row as String arrays. This can be used for smaller datapackages, + * but for huge or unknown sizes, reading via iterator is preferred, as this method loads all data into RAM. + * + * It can be configured to return table rows with relations to other data sources resolved + * + * The method uses Iterators provided by {@link Table} class, and is roughly implemented after + * https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py + * + * @param relations true: follow relations + * @return A list of table rows. + * @throws Exception if parsing the data fails * - * @param outputFile the file to write to. - * @param dialect the CSV dialect to use for writing - * @throws Exception if something fails while writing */ - void writeTableAsCsv(Table table, Dialect dialect, Path outputFile) throws Exception; + @JsonIgnore + public List getData(boolean relations) throws Exception; /** - * Returns an Iterator that returns rows as object-arrays - * @return - * @throws Exception + * Read all data from a Resource, each row as Map objects. This can be used for smaller datapackages, + * but for huge or unknown sizes, reading via iterator is preferred, as this method loads all data into RAM. + * + * The method returns Map<String,Object> where key is the header name, and val is the data. + * It can be configured to return table rows with relations to other data sources resolved + * + * The method uses Iterators provided by {@link Table} class, and is roughly implemented after + * https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py + * + * @param relations true: follow relations + * @return A list of table rows. + * @throws Exception if parsing the data fails + * */ - Iterator objectArrayIterator() throws Exception; + List> getMappedData(boolean relations) throws Exception; /** - * Returns an Iterator that returns rows as object-arrays - * @return - * @throws Exception + * Most customizable method to retrieve all data in a Resource. Parameters match those in + * {@link io.frictionlessdata.tableschema.Table#iterator(boolean, boolean, boolean, boolean)}. + * This can be used for smaller datapackages, but for huge or unknown + * sizes, reading via iterator is preferred, as this method loads all data into RAM. + * + * The method can be configured to return table rows as: + *
    + *
  • String arrays (parameter `cast` = false)
    • + *
  • as Object arrays (parameter `cast` = true)
    • + *
  • as a Map<String,Object> where key is the header name, and val is + * the data (parameter `keyed` = true)
    • + *
  • in an "extended" form (parameter `extended` = true) that returns an Object array where the first entry + * is the row number, the second is a String array holding the headers, + * and the third is an Object array holding the row data.
    • + *
  • with relations to other data sources resolved
    • + *
+ * + * The method uses Iterators provided by {@link Table} class, and is roughly implemented after + * https://github.com/frictionlessdata/tableschema-py/blob/master/tableschema/table.py + * + * @param keyed true: return table rows as key/value maps + * @param extended true: return table rows in an extended form + * @param cast true: convert CSV cells to Java objects other than String + * @param relations true: follow relations + * @return A list of table rows. + * @throws Exception if parsing the data fails + * */ - Iterator objectArrayIterator(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception; + List getData(boolean keyed, boolean extended, boolean cast, boolean relations) throws Exception; /** - * Returns an Iterator that returns rows as string-arrays - * @return - * @throws Exception + * Read all data from a Resource. This can be used for smaller datapackages, but for huge or unknown + * sizes, reading via iterator is preferred, as this method loads all data into RAM. + * The method ignores relations. + * + * Returns as a List of Java objects of the type `beanClass`. Under the hood, it uses a {@link TableIterator} + * for reading based on a Java Bean class instead of a {@link io.frictionlessdata.tableschema.schema.Schema}. + * It therefore disregards the Schema set on the {@link io.frictionlessdata.tableschema.Table} the iterator works + * on but creates its own Schema from the supplied `beanType`. + * + * @return List of rows as bean instances. + * @param beanClass the Bean class this BeanIterator expects */ - public Iterator stringArrayIterator() throws Exception; - - String[] getHeaders() throws Exception; + List getData(Class beanClass) throws Exception; /** - * @return the name + * Read all data from all Tables and return it as JSON. + * + * It ignores relations to other data sources. + * + * @return A JSON representation of the data as a String. */ - String getName(); + @JsonIgnore + String getDataAsJson(); /** - * @param name the name to set + * Read all data from all Tables and return it as a String in the format of the Resource's Dialect. + * Column order will be deducted from the table data source. + * + * @return A CSV representation of the data as a String. */ - void setName(String name); + @JsonIgnore + String getDataAsCsv(); /** - * @return the profile + * Return the data of all Tables as a CSV string, + * + * - the `dialect` parameter decides on the CSV options. If it is null, then the file will + * be written as RFC 4180 compliant CSV + * - the `schema` parameter decides on the order of the headers in the CSV file. If it is null, + * the Schema of the Resource will be used, or if none, the order of the columns will be + * the same as in the Tables. + * + * It ignores relations to other data sources. + * + * @param dialect the CSV dialect to use + * @param schema a Schema defining header row names in the order in which data should be exported + * + * @return A CSV representation of the data as a String. */ - String getProfile(); + String getDataAsCsv(Dialect dialect, Schema schema); /** - * @param profile the profile to set + * Write all the data in this resource into one or more + * files inside `outputDir`, depending on how many tables this + * Resource holds. + * + * @param outputDir the directory to write to. Code must create + * files as needed. + * @throws Exception if something fails while writing */ - void setProfile(String profile); + void writeData(Path outputDir) throws Exception; /** - * @return the title + * Write all the data in this resource into the provided {@link Writer}. + * + * @param out the writer to write to. + * @throws Exception if something fails while writing */ - String getTitle(); + void writeData(Writer out) throws Exception; /** - * @param title the title to set + * Write the Resource {@link Schema} to `outputDir`. + * + * @param parentFilePath the directory to write to. Code must create + * files as needed. + * @throws IOException if something fails while writing */ - void setTitle(String title); + void writeSchema(Path parentFilePath) throws IOException; /** - * @return the description + * Write the Resource {@link Dialect} to `outputDir`. + * + * @param parentFilePath the directory to write to. Code must create + * files as needed. + * @throws IOException if something fails while writing */ - String getDescription(); + void writeDialect(Path parentFilePath) throws IOException; /** - * @param description the description to set + * Returns an Iterator that returns rows as object-arrays. Values in each column + * are parsed and converted ("cast") to Java objects based on the Field definitions of the Schema. + * @return Iterator returning table rows as Object Arrays + * @throws Exception if parsing the data fails */ - void setDescription(String description); - + Iterator objectArrayIterator() throws Exception; /** - * @return the mediaType + * Returns an Iterator that returns rows as object-arrays. Values in each column + * are parsed and converted ("cast") to Java objects based on the Field definitions of the Schema. + * @return Iterator returning table rows as Object Arrays + * @throws Exception if parsing the data fails */ - String getMediaType(); + Iterator objectArrayIterator(boolean extended, boolean relations) throws Exception; /** - * @param mediaType the mediaType to set + * Returns an Iterator that returns rows as a Map<key,val> where key is the header name, and val is the data. + * It can be configured to follow relations + * + * @param relations Whether references to other data sources get resolved + * @return Iterator that returns rows as Maps. + * @throws Exception if parsing the data fails */ - void setMediaType(String mediaType); + Iterator> mappingIterator(boolean relations) throws Exception; /** - * @return the encoding + * Returns an Iterator that returns rows as bean-arrays. + * {@link TableIterator} based on a Java Bean class instead of a {@link io.frictionlessdata.tableschema.schema.Schema}. + * It therefore disregards the Schema set on the {@link io.frictionlessdata.tableschema.Table} the iterator works + * on but creates its own Schema from the supplied `beanType`. + * + * @return Iterator that returns rows as bean-arrays. + * @param beanType the Bean class this BeanIterator expects + * @param relations follow relations to other data source */ - String getEncoding(); + Iterator beanIterator(Class beanType, boolean relations)throws Exception; /** - * @param encoding the encoding to set + * This method creates an Iterator that will return table rows as String arrays. + * It therefore disregards the Schema set on the table. It does not follow relations. + * + * @return Iterator that returns rows as string arrays. */ - void setEncoding(String encoding); + public Iterator stringArrayIterator() throws Exception; /** - * @return the bytes + * This method creates an Iterator that will return table rows as String arrays. + * It therefore disregards the Schema set on the table. It can be configured to follow relations. + * + * @return Iterator that returns rows as string arrays. */ - Integer getBytes(); + public Iterator stringArrayIterator(boolean relations) throws Exception; + + + String[] getHeaders() throws Exception; /** - * @param bytes the bytes to set + * Construct a path to write out the Schema for this Resource + * @return a String containing a relative path for writing or null */ - void setBytes(Integer bytes); + String getPathForWritingSchema(); /** - * @return the hash + * Construct a path to write out the Dialect for this Resource + * @return a String containing a relative path for writing or null */ - String getHash(); + String getPathForWritingDialect(); /** - * @param hash the hash to set + * Return a set of relative path names we would use if we wanted to write + * the resource data to file. For DataResources, this helps with conversion + * to FileBasedResources + * @return Set of relative path names */ - void setHash(String hash); + Set getDatafileNamesForWriting(); /** * @return the dialect @@ -188,70 +340,114 @@ public interface Resource { */ void setFormat(String format); - String getSchemaReference(); - String getDialectReference(); Schema getSchema(); void setSchema(Schema schema); - /** - * @return the sources - */ - JSONArray getSources(); + public Schema inferSchema() throws TypeInferringException; - /** - * @param sources the sources to set - */ - void setSources(JSONArray sources); + @JsonIgnore + boolean shouldSerializeToFile(); - /** - * @return the licenses - */ - JSONArray getLicenses(); + @JsonIgnore + void setShouldSerializeToFile(boolean serializeToFile); /** - * @param licenses the licenses to set + * Sets the format (either CSV or JSON) for serializing the Resource content to File. + * @param format either FORMAT_CSV or FORMAT_JSON, other strings will cause an Exception */ - void setLicenses(JSONArray licenses); + void setSerializationFormat(String format); + String getSerializationFormat(); + void checkRelations(Package pkg) throws Exception; - static AbstractResource build(JSONObject resourceJson, Object basePath, boolean isArchivePackage) throws IOException, DataPackageException, Exception { - String name = resourceJson.has(JSONBase.JSON_KEY_NAME) ? resourceJson.getString(JSONBase.JSON_KEY_NAME) : null; - Object path = resourceJson.has(JSONBase.JSON_KEY_PATH) ? resourceJson.get(JSONBase.JSON_KEY_PATH) : null; - Object data = resourceJson.has(JSONBase.JSON_KEY_DATA) ? resourceJson.get(JSONBase.JSON_KEY_DATA) : null; - String format = resourceJson.has(JSONBase.JSON_KEY_FORMAT) ? resourceJson.getString(JSONBase.JSON_KEY_FORMAT) : null; + /** + * Recreate a Resource object from a JSON descriptor, a base path to resolve relative file paths against + * and a flag that tells us whether we are reading from inside a ZIP archive. + * + * @param resourceJson JSON descriptor containing properties like `name, `data` or `path` + * @param basePath File system path used to resolve relative path entries if `path` contains entries + * @param isArchivePackage true if we are reading files from inside a ZIP archive. + * @return fully inflated Resource object. Subclass depends on the data found + * @throws IOException thrown if reading data failed + * @throws DataPackageException for invalid data + * @throws Exception if other operation fails. + */ + + static AbstractResource fromJSON( + ObjectNode resourceJson, + Object basePath, + boolean isArchivePackage) throws IOException, DataPackageException, Exception { + String name = textValueOrNull(resourceJson, JSONBase.JSON_KEY_NAME); + Object path = resourceJson.get(JSONBase.JSON_KEY_PATH); + Object data = resourceJson.get(JSON_KEY_DATA); + String format = textValueOrNull(resourceJson, JSONBase.JSON_KEY_FORMAT); + String profile = textValueOrNull(resourceJson, JSONBase.JSON_KEY_PROFILE); Dialect dialect = JSONBase.buildDialect (resourceJson, basePath, isArchivePackage); Schema schema = JSONBase.buildSchema(resourceJson, basePath, isArchivePackage); + String encoding = textValueOrNull(resourceJson, JSONBase.JSON_KEY_ENCODING); + Charset charset = TableDataSource.getDefaultEncoding(); + if (StringUtils.isNotEmpty(encoding)) { + charset = Charset.forName(encoding); + } // Now we can build the resource objects AbstractResource resource = null; if (path != null){ Collection paths = fromJSON(path, basePath); - resource = build(name, paths, basePath); + resource = build(name, paths, basePath, charset); + resource.setFormat(format); if (resource instanceof FilebasedResource) { ((FilebasedResource)resource).setIsInArchive(isArchivePackage); } - } else if (data != null && format != null){ - if (format.equals(Resource.FORMAT_JSON)) - resource = new JSONDataResource(name, ((JSONArray) data).toString()); - else if (format.equals(Resource.FORMAT_CSV)) - resource = new CSVDataResource(name, (String)data); + // inlined data + } else if (data != null){ + if (null == format) { + resource = buildJsonResource(data, name, null, profile); + } else if (format.equals(Resource.FORMAT_JSON)) + resource = buildJsonResource(data, name, format, profile); + else if (format.equals(Resource.FORMAT_CSV)) { + // data is in inline CSV format like "data": "A,B,C\n1,2,3\n4,5,6" + String dataString = ((StringNode)data).stringValue().replaceAll("\\\\n", "\n"); + resource = new CSVDataResource(name, dataString); + } } else { - DataPackageException dpe = new DataPackageException( + throw new DataPackageValidationException( "Invalid Resource. The path property or the data and format properties cannot be null."); - throw dpe; } resource.setDialect(dialect); - JSONBase.setFromJson(resourceJson, resource, schema); + JSONBase.setFromJson(resourceJson, resource); + resource.setSchema(schema); return resource; } + private static AbstractResource buildJsonResource(Object data, String name, String format, String profile) { + AbstractResource resource = null; + if ((data instanceof ArrayNode)) { + resource = new JSONDataResource(name, (ArrayNode)data); + } else { + if ((null != profile) && profile.equalsIgnoreCase(Profile.PROFILE_TABULAR_DATA_RESOURCE) && (StringUtils.isEmpty(format))) { + // from the spec: " a JSON string - in this case the format or + // mediatype properties MUST be provided + // https://specs.frictionlessdata.io/data-resource/#data-inline-data + throw new DataPackageValidationException( + "Invalid Resource. The format property cannot be null for inlined CSV data."); + } else if ((data instanceof ObjectNode)) { + resource = new JSONObjectResource(name, (ObjectNode)data); + } else { + throw new DataPackageValidationException( + "Invalid Resource. No implementation for inline data of type " + data.getClass().getSimpleName()); + } + } + return resource; + } - static AbstractResource build(String name, Collection pathOrUrl, Object basePath) throws MalformedURLException { + static AbstractResource build(String name, Collection pathOrUrl, Object basePath, Charset encoding) + throws MalformedURLException { if (pathOrUrl != null) { List files = new ArrayList<>(); List urls = new ArrayList<>(); @@ -263,8 +459,8 @@ static AbstractResource build(String name, Collection pathOrUrl, Object basePath files.add(((Path)o).toFile()); } else if (o instanceof URL) { urls.add((URL)o); - } else if (o instanceof String) { - strings.add((String)o); + } else if (o instanceof StringNode) { + strings.add(o.toString()); } else { throw new IllegalArgumentException("Cannot build a resource out of "+o.getClass()); } @@ -275,7 +471,7 @@ static AbstractResource build(String name, Collection pathOrUrl, Object basePath for (String s : strings) { if (basePath instanceof URL) { /* - * We have a URL fragment, that is not valid on its own. + * We have a URL fragment that is not valid on its own. * According to https://github.com/frictionlessdata/specs/issues/652 , * URL fragments should be resolved relative to the base URL */ @@ -299,7 +495,7 @@ static AbstractResource build(String name, Collection pathOrUrl, Object basePath if (!files.isEmpty() && !urls.isEmpty()) { throw new DataPackageException("Resources with mixed URL/File paths are not allowed"); } else if (!files.isEmpty()) { - return new FilebasedResource(name, files, normalizePath(basePath)); + return new FilebasedResource(name, files, normalizePath(basePath), encoding); } else if (!urls.isEmpty()) { return new URLbasedResource(name, urls); } @@ -326,43 +522,43 @@ static File normalizePath(Object basePath) { static Collection fromJSON(Object path, Object basePath) throws IOException { if (null == path) return null; - Collection paths; - if (path instanceof JSONArray) { - paths = fromJSON((JSONArray) path, basePath); + if (path instanceof ArrayNode) { + return fromJSON((ArrayNode) path, basePath); + } else if (path instanceof StringNode) { + return fromJSON(JsonUtil.getInstance().createArrayNode().add((StringNode)path), basePath); } else { - paths = new ArrayList(); - paths.add(path); + return Collections.singleton(path); } - return paths; } - static Collection fromJSON(JSONArray arr, Object basePath) throws IOException { + static Collection fromJSON(ArrayNode arr, Object basePath) throws IOException { if (null == arr) return null; Collection dereferencedObj = new ArrayList(); - for (Object obj : arr) { - if (!(obj instanceof String)) + for (JsonNode obj : arr) { + if (!(obj.isString())) throw new IllegalArgumentException("Cannot dereference a "+obj.getClass()); - if (isValidUrl((String)obj)) { + String location = obj.asString(); + if (isValidUrl(location)) { /* This is a fully qualified URL "https://somesite.com/data/cities.csv". */ - dereferencedObj.add(new URL((String)obj)); + dereferencedObj.add(new URL(location)); } else { if (basePath instanceof Path) { /* relative path, store for later dereferencing. For reading, must be read relative to the basePath */ - dereferencedObj.add(new File((String) obj)); + dereferencedObj.add(new File(location)); } else if (basePath instanceof URL) { /* This is a URL fragment "data/cities.csv". According to https://github.com/frictionlessdata/specs/issues/652, it should be parsed against the base URL (the Descriptor URL) */ - dereferencedObj.add(new URL(((URL)basePath),(String)obj)); + dereferencedObj.add(new URL(((URL)basePath),location)); } } } @@ -396,4 +592,10 @@ static Path toSecure(Path testPath, Path referencePath) throws IOException { return resolvedPath; } + + static String textValueOrNull(JsonNode source, String fieldName) { + return source.has(fieldName) ? source.get(fieldName).asString() : null; + } + + void validate(Package pkg) throws DataPackageValidationException; } \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/ResourceBuilder.java b/src/main/java/io/frictionlessdata/datapackage/resource/ResourceBuilder.java new file mode 100644 index 0000000..cc46d67 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/resource/ResourceBuilder.java @@ -0,0 +1,361 @@ +package io.frictionlessdata.datapackage.resource; + +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Dialect; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Builder for creating different types of Resources with a fluent API + */ +public class ResourceBuilder { + private String name; + private String format; + private String profile; + private Schema schema; + private Dialect dialect; + private String title; + private String description; + private String encoding; + private String mediaType; + private boolean serializeToFile = false; + private String serializationFormat; + + private boolean shouldInferSchema = false; + + // Data source fields + private Object data; + private List paths; + private List urls; + private File basePath; + + private ResourceBuilder(String name) { + this.name = name; + } + + /** + * Start building a new Resource with the given name + */ + public static ResourceBuilder create(String name) { + return new ResourceBuilder(name); + } + + /** + * Sniff string data for the data type + */ + public ResourceBuilder withData(String data) { + try { + JsonNode json = JsonUtil.getInstance().readValue(data); + this.format = Resource.FORMAT_JSON; + if (json instanceof ArrayNode) { + withJsonArrayData((ArrayNode)json); + } else { + withJsonObjectData((ObjectNode) json); + } + } catch (Exception ex) { + // might be CSV or other format, check later + this.data = data; + } + + return this; + } + + /** + * Set CSV string data for the resource + */ + public ResourceBuilder withCsvData(String csvData) { + this.data = csvData; + this.format = Resource.FORMAT_CSV; + return this; + } + + /** + * Set JSON array data for the resource + */ + public ResourceBuilder withJsonArrayData(ArrayNode jsonArray) { + this.data = jsonArray; + this.format = Resource.FORMAT_JSON; + return this; + } + + /** + * Set JSON object data for non-tabular resource + */ + public ResourceBuilder withJsonObjectData(ObjectNode jsonObject) { + this.data = jsonObject; + this.format = Resource.FORMAT_JSON; + this.profile = Profile.PROFILE_DATA_RESOURCE_DEFAULT; + return this; + } + + /** + * Set files for the resource + */ + public ResourceBuilder withFiles(File basePath, Collection paths) { + this.paths = new ArrayList<>(paths); + this.basePath = basePath; + return this; + } + + /** + * Set file paths for the resource + */ + public ResourceBuilder withFiles(File basePath, String... paths) { + this.paths = Arrays.stream(paths).map(File::new).collect(Collectors.toList()); + this.basePath = basePath; + return this; + } + + /** + * Set a single file path for the resource + */ + public ResourceBuilder withFile(File basePath, String path) { + if (null == this.paths) { + this.paths = new ArrayList<>(); + } + this.paths.add(new File(path)); + this.basePath = basePath; + return this; + } + + /** + * Set a single file for the resource + */ + public ResourceBuilder withFile(File basePath, File path) { + if (null == this.paths) { + this.paths = new ArrayList<>(); + } + this.paths.add(path); + this.basePath = basePath; + return this; + } + + /** + * Set URLs for the resource + */ + public ResourceBuilder withUrls(List urls) { + this.urls = new ArrayList<>(urls); + return this; + } + + /** + * Set a single URL for the resource + */ + public ResourceBuilder withUrl(URL url) { + this.urls = new ArrayList<>(); + this.urls.add(url); + return this; + } + + /** + * Set custom format (overrides auto-detection) + */ + public ResourceBuilder format(String format) { + this.format = format; + return this; + } + + /** + * Set the profile (default: tabular-data-resource for tabular data) + */ + public ResourceBuilder profile(String profile) { + this.profile = profile; + return this; + } + + /** + * Set the schema + */ + public ResourceBuilder schema(Schema schema) { + this.schema = schema; + return this; + } + + /** + * Infer the schema from the data source. + */ + public ResourceBuilder inferSchema() { + this.shouldInferSchema = true; + return this; + } + + /** + * Set the dialect + */ + public ResourceBuilder dialect(Dialect dialect) { + this.dialect = dialect; + return this; + } + + /** + * Set whether to serialize inline data to file when saving package + */ + public ResourceBuilder serializeToFile(boolean serialize) { + this.serializeToFile = serialize; + return this; + } + + /** + * Set the format for serialization (csv or json) + */ + public ResourceBuilder serializationFormat(String format) { + this.serializationFormat = format; + return this; + } + + /** + * Set the title + */ + public ResourceBuilder title(String title) { + this.title = title; + return this; + } + + /** + * Set the description + */ + public ResourceBuilder description(String description) { + this.description = description; + return this; + } + + /** + * Set the encoding + */ + public ResourceBuilder encoding(String encoding) { + this.encoding = encoding; + return this; + } + + /** + * Set the media type + */ + public ResourceBuilder mediaType(String mediaType) { + this.mediaType = mediaType; + return this; + } + + + + /** + * Build the Resource instance + */ + public Resource build() { + Resource resource = null; + + if (null == encoding) { + encoding = TableDataSource.getDefaultEncoding().toString(); + } + Charset charset = Charset.forName(encoding); + + if (shouldInferSchema) { + if (null != schema) { + throw new IllegalStateException("Cannot infer schema when schema is already provided"); + } + if (null != data) { + schema = Schema.infer(data, charset); + } else if (null != paths) { + List files = paths.stream().map((f) -> new File(basePath, f.getPath())).collect(Collectors.toList()); + schema = Schema.infer(files, charset); + } else if (null != urls) { + schema = Schema.infer(urls, charset); + } + } + + // Create appropriate resource type based on provided data + if (data != null) { + if (null == profile) { + profile = Profile.PROFILE_TABULAR_DATA_RESOURCE; + } + if (format != null && format.equals(Resource.FORMAT_CSV)) { + resource = new CSVDataResource(name, (String) data); + } else if (format != null && format.equals(Resource.FORMAT_JSON)) { + if (data instanceof String) { + resource = new JSONDataResource(name, (String) data); + } else if (data instanceof ObjectNode) { + resource = new JSONObjectResource(name, (ObjectNode)data); + } else if (data instanceof ArrayNode) { + resource = new JSONDataResource(name, (ArrayNode)data); + } + } else { + try { + CSVParser.parse((String)data, TableDataSource.getDefaultCsvFormat()).getHeaderMap(); + this.format = Resource.FORMAT_CSV; + } catch (Exception ex2) { + throw new IllegalStateException("Cannot determine resource type from data, neigher JSON nor CSV format detected"); + } + throw new IllegalStateException("Cannot determine resource type from data"); + } + } else if (paths != null && !paths.isEmpty()) { + try { + resource = Resource.build(name, paths, basePath, charset); + } catch (Exception e) { + throw new DataPackageException(e); + } + } else if (urls != null && !urls.isEmpty()) { + try { + resource = Resource.build(name, urls, null, charset); + } catch (Exception e) { + throw new DataPackageException(e); + } + } else { + throw new IllegalStateException("No data source provided for resource"); + } + + // Set common properties + if (profile != null) { + resource.setProfile(profile); + } + if (schema != null) { + resource.setSchema(schema); + } + if (dialect != null) { + resource.setDialect(dialect); + } + if (title != null) { + resource.setTitle(title); + } + if (description != null) { + resource.setDescription(description); + } + if (encoding != null) { + resource.setEncoding(encoding); + } + if (mediaType != null) { + resource.setMediaType(mediaType); + } + if (format != null && resource.getFormat() == null) { + resource.setFormat(format); + } + + resource.setShouldSerializeToFile(serializeToFile); + if (serializationFormat != null) { + resource.setSerializationFormat(serializationFormat); + } + + return resource; + } + +} \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java index 14cd96a..71a75a1 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java @@ -1,23 +1,21 @@ package io.frictionlessdata.datapackage.resource; -import io.frictionlessdata.datapackage.Dialect; -import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import com.fasterxml.jackson.annotation.JsonInclude; import io.frictionlessdata.tableschema.Table; -import io.frictionlessdata.tableschema.datasourceformats.DataSourceFormat; -import org.apache.commons.csv.CSVFormat; +import java.io.IOException; +import java.io.InputStream; import java.net.URL; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import static io.frictionlessdata.datapackage.Package.isValidUrl; - -public class URLbasedResource extends AbstractReferencebasedResource { +@JsonInclude(value= JsonInclude.Include. NON_EMPTY, content= JsonInclude.Include. NON_NULL) +public class URLbasedResource extends AbstractReferencebasedResource { public URLbasedResource(String name, Collection paths) { super(name, paths); + serializeToFile = false; } @Override @@ -25,6 +23,13 @@ Table createTable(URL reference) throws Exception { return Table.fromSource(reference, schema, getCsvFormat()); } + @Override + byte[] getRawData(URL input) throws IOException { + try (InputStream inputStream = input.openStream()) { + return getRawData(inputStream); + } + } + @Override String getStringRepresentation(URL reference) { return reference.toExternalForm(); @@ -42,25 +47,4 @@ List
readData () throws Exception{ } return tables; } - - - @Override - public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception { - Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; - List paths = new ArrayList<>(getReferencesAsStrings()); - int cnt = 0; - for (String path : paths) { - String fileName; - if (isValidUrl(path)) { - URL url = new URL (path); - fileName = url.getFile(); - } else { - throw new DataPackageException("Cannot writeDataAsCsv for "+path); - } - List
tables = getTables(); - Table t = tables.get(cnt++); - Path p = outputDir.resolve(fileName); - writeTableAsCsv(t, lDialect, p); - } - } } diff --git a/src/main/resources/schemas/tabular-data-package.json b/src/main/resources/schemas/tabular-data-package.json index f6d4d9d..1498e36 100755 --- a/src/main/resources/schemas/tabular-data-package.json +++ b/src/main/resources/schemas/tabular-data-package.json @@ -2014,7 +2014,17 @@ "propertyOrder": 50, "title": "CSV Dialect", "description": "The CSV dialect descriptor.", - "type": "object", + "oneOf": [ + { + "title": "Dialect file path", + "description": "A fully qualified URL, or a POSIX file path.", + "type": "string" + }, + { + "title": "Dialect as JSON", + "description": "A dialect encoded as a JSON object", + "type": "object" + }], "required": [ "delimiter", "doubleQuote" diff --git a/src/main/resources/schemas/tabular-data-resource.json b/src/main/resources/schemas/tabular-data-resource.json index def495e..3270d54 100755 --- a/src/main/resources/schemas/tabular-data-resource.json +++ b/src/main/resources/schemas/tabular-data-resource.json @@ -1782,7 +1782,17 @@ "propertyOrder": 50, "title": "CSV Dialect", "description": "The CSV dialect descriptor.", - "type": "object", + "oneOf": [ + { + "title": "Dialect file path", + "description": "A fully qualified URL, or a POSIX file path.", + "type": "string" + }, + { + "title": "Dialect as JSON", + "description": "A dialect encoded as a JSON object", + "type": "object" + }], "required": [ "delimiter", "doubleQuote" diff --git a/src/test/java/io/frictionlessdata/datapackage/ContributorTest.java b/src/test/java/io/frictionlessdata/datapackage/ContributorTest.java new file mode 100644 index 0000000..f962dd0 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/ContributorTest.java @@ -0,0 +1,65 @@ +package io.frictionlessdata.datapackage; + +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Collection; + +public class ContributorTest { + + String validContributorsJson = + "[{" + + "\"title\":\"HDIC\"," + + "\"email\":\"me@example.com\"," + + "\"path\":\"https://example.com\"," + + "\"role\":\"author\"," + + "\"organization\":\"space cadets\"" + + "}]"; + + String invalidPathContributorsJson = + "[{" + + "\"title\":\"HDIC\"," + + "\"email\":\"me@example.com\"," + + "\"path\":\"qwerty\"," + + "\"role\":\"author\"," + + "\"organization\":\"space cadets\"" + + "}]"; + + String invalidRoleContributorsJson = + "[{" + + "\"title\":\"HDIC\"," + + "\"email\":\"me@example.com\"," + + "\"path\":\"https://example.com\"," + + "\"role\":\"ERTYUIJHG\"," + + "\"organization\":\"space cadets\"" + + "}]"; + + @Test + @DisplayName("validate serialization and deserialization of a contributor object") + public void testSerialization() { + Collection contributors = Contributor.fromJson(validContributorsJson); + JsonUtil instance = JsonUtil.getInstance(); + String actual = instance.serialize(contributors, false); + Assertions.assertEquals(validContributorsJson, actual); + } + + @Test + @DisplayName("validate DPE is thrown with invalid url") + public void testInvalidPath() { + DataPackageException ex = Assertions.assertThrows(DataPackageException.class, ()->{ + Contributor.fromJson(invalidPathContributorsJson); + }); + Assertions.assertEquals(Contributor.invalidUrlMsg, ex.getMessage()); + } + + @Test + @DisplayName("validate Roles can be any string") + // fix for https://github.com/frictionlessdata/datapackage-java/issues/45 after frictionless changed spec + public void testInvalidRole() { + Collection contributors = Contributor.fromJson(invalidRoleContributorsJson); + Assertions.assertEquals(contributors.iterator().next().getRole(), "ERTYUIJHG"); + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/DialectTest.java b/src/test/java/io/frictionlessdata/datapackage/DialectTest.java index 6718004..949707e 100644 --- a/src/test/java/io/frictionlessdata/datapackage/DialectTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/DialectTest.java @@ -11,7 +11,7 @@ class DialectTest { private Character quoteChar ='"'; private Boolean doubleQuote = true; private Character escapeChar = null; - private String nullSequence = null; + private String nullSequence = ""; private Boolean skipInitialSpace = true; private Boolean hasHeaderRow = true; private Character commentChar = null; @@ -57,4 +57,42 @@ void testDialectFromJson() { Assertions.assertEquals('#', dia.getCommentChar()); Assertions.assertFalse(dia.isHasHeaderRow()); } + + @Test + @DisplayName("clone Dialect") + void testCloneDialect() throws CloneNotSupportedException { + String json = "{ "+ + " \"delimiter\":\"\t\", "+ + " \"header\":\"false\", "+ + " \"quoteChar\":\"\\\"\", " + + " \"commentChar\":\"#\" "+ + "}"; + Dialect dia = Dialect.fromJson(json); + Dialect clone = dia.clone(); + Assertions.assertEquals(dia, clone); + } + + @Test + @DisplayName("Hashcode for Dialect") + void testDialectHashCode() { + String json = "{ "+ + " \"delimiter\":\"\t\", "+ + " \"header\":\"false\", "+ + " \"quoteChar\":\"\\\"\", " + + " \"commentChar\":\"#\" "+ + "}"; + Dialect dia = Dialect.fromJson(json); + + Assertions.assertEquals(2019600303, dia.hashCode()); + Assertions.assertEquals(-1266124318, Dialect.DEFAULT.hashCode()); + } + + @Test + @DisplayName("Hashcode for Dialect") + void testDefaultDialectJson() { + String defaultJson = "{\"caseSensitiveHeader\":false,\"quoteChar\":\"\\\"\",\"doubleQuote\":true," + + "\"delimiter\":\",\",\"lineTerminator\":\"\\r\\n\",\"nullSequence\":\"\"," + + "\"header\":true,\"csvddfVersion\":1.2,\"skipInitialSpace\":true}"; + Assertions.assertEquals(defaultJson, Dialect.DEFAULT.asJson()); + } } diff --git a/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java b/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java new file mode 100644 index 0000000..d2964e8 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/DocumentationCases.java @@ -0,0 +1,69 @@ +package io.frictionlessdata.datapackage; + +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.datapackage.resource.Resource; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Paths; + +public class DocumentationCases { + + @Test + @DisplayName("Reading a Schema with a Foreign Key against non-matching data") + void validateForeignKeyWithError() throws Exception{ + String DESCRIPTOR = "{\n" + + " \"name\": \"foreign-keys\",\n" + + " \"resources\": [\n" + + " {\n" + + " \"name\": \"teams\",\n" + + " \"data\": [\n" + + " [\"id\", \"name\", \"city\"],\n" + + " [\"1\", \"Arsenal\", \"London\"],\n" + + " [\"2\", \"Real\", \"Madrid\"],\n" + + " [\"3\", \"Bayern\", \"Munich\"]\n" + + " ],\n" + + " \"schema\": {\n" + + " \"fields\": [\n" + + " {\n" + + " \"name\": \"id\",\n" + + " \"type\": \"integer\"\n" + + " },\n" + + " {\n" + + " \"name\": \"name\",\n" + + " \"type\": \"string\"\n" + + " },\n" + + " {\n" + + " \"name\": \"city\",\n" + + " \"type\": \"string\"\n" + + " }\n" + + " ],\n" + + " \"foreignKeys\": [\n" + + " {\n" + + " \"fields\": \"city\",\n" + + " \"reference\": {\n" + + " \"resource\": \"cities\",\n" + + " \"fields\": \"name\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " },\n" + + " {\n" + + " \"name\": \"cities\",\n" + + " \"data\": [\n" + + " [\"name\", \"country\"],\n" + + " [\"London\", \"England\"],\n" + + " [\"Madrid\", \"Spain\"]\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + + Package dp = new Package(DESCRIPTOR, Paths.get(""), true); + Resource teams = dp.getResource("teams"); + DataPackageValidationException dpe = Assertions.assertThrows(DataPackageValidationException.class, () -> teams.checkRelations(dp)); + Assertions.assertEquals("Error reading data with relations: Foreign key validation failed: [city] -> [name]: 'Munich' not found in resource 'cities'.", dpe.getMessage()); + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/PackageTest.java b/src/test/java/io/frictionlessdata/datapackage/PackageTest.java index 129f291..78b1e0a 100644 --- a/src/test/java/io/frictionlessdata/datapackage/PackageTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/PackageTest.java @@ -1,68 +1,61 @@ package io.frictionlessdata.datapackage; +import tools.jackson.databind.JsonNode; +import tools.jackson.databind.node.ArrayNode; import io.frictionlessdata.datapackage.beans.EmployeeBean; import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageFileOrUrlNotFoundException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.datapackage.resource.*; +import io.frictionlessdata.tableschema.exception.ConstraintsException; +import io.frictionlessdata.tableschema.exception.TableValidationException; +import io.frictionlessdata.tableschema.exception.ValidationException; +import io.frictionlessdata.tableschema.field.DateField; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + import java.io.File; import java.io.FileNotFoundException; -import java.io.IOException; +import java.io.FileWriter; +import java.math.BigDecimal; +import java.math.BigInteger; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.security.MessageDigest; +import java.time.ZonedDateTime; import java.util.*; -import io.frictionlessdata.datapackage.exceptions.DataPackageFileOrUrlNotFoundException; -import io.frictionlessdata.datapackage.resource.JSONDataResource; -import io.frictionlessdata.datapackage.resource.FilebasedResource; -import io.frictionlessdata.datapackage.resource.Resource; -import io.frictionlessdata.datapackage.resource.ResourceTest; -import io.frictionlessdata.tableschema.field.DateField; -import io.frictionlessdata.tableschema.schema.Schema; -import io.frictionlessdata.tableschema.Table; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; -import org.junit.rules.ExpectedException; -import org.junit.Assert; -import org.junit.rules.TemporaryFolder; - -import javax.json.JsonObject; - +import static io.frictionlessdata.datapackage.Profile.*; import static io.frictionlessdata.datapackage.TestUtil.getBasePath; +import static org.junit.jupiter.api.Assertions.assertThrows; + /** * * */ public class PackageTest { + private static boolean verbose = false; private static URL validUrl; - static JSONObject resource1 = new JSONObject("{\"name\": \"first-resource\", \"path\": " + - "[\"data/cities.csv\", \"data/cities2.csv\", \"data/cities3.csv\"]}"); - static JSONObject resource2 = new JSONObject("{\"name\": \"second-resource\", \"path\": " + - "[\"data/area.csv\", \"data/population.csv\"]}"); + static String resource1String = "{\"name\": \"first-resource\", \"path\": " + + "[\"data/cities.csv\", \"data/cities2.csv\", \"data/cities3.csv\"]}"; + static String resource2String = "{\"name\": \"second-resource\", \"path\": " + + "[\"data/area.csv\", \"data/population.csv\"]}"; - static JSONArray testResources; + static ArrayNode testResources = JsonUtil.getInstance().createArrayNode(String.format("[%s,%s]", resource1String, resource2String)); - static { - testResources = new JSONArray(); - testResources.put(resource1); - testResources.put(resource2); - } - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Rule - public final ExpectedException exception = ExpectedException.none(); - - @Before - public void setup() throws MalformedURLException { + @BeforeAll + public static void setup() throws MalformedURLException { validUrl = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/datapackages/multi-data/datapackage.json"); } @@ -74,82 +67,77 @@ public void testLoadFromJsonString() throws Exception { Package dp = this.getDataPackageFromFilePath(true); // Assert - Assert.assertNotNull(dp); + Assertions.assertNotNull(dp); } @Test - public void testLoadFromValidJsonObject() throws Exception { - // Create JSON Object for testing - JSONObject jsonObject = new JSONObject("{\"name\": \"test\"}"); - - List resourceArrayList = new ArrayList(); - resourceArrayList.add(resource1); - resourceArrayList.add(resource2); + public void testLoadFromValidJsonNode() throws Exception { + // Create Object for testing + Map testMap = createTestMap(); - JSONArray resources = new JSONArray(resourceArrayList); - // Add the resources - jsonObject.put("resources", resources); + testMap.put("resources", testResources); - // Build the datapackage - Package dp = new Package(jsonObject.toString(), getBasePath(), true); + // convert the object to a json string and Build the datapackage + Package dp = new Package(asString(testMap), getBasePath(), true); // Assert - Assert.assertNotNull(dp); + Assertions.assertNotNull(dp); } @Test - public void testLoadFromValidJsonObjectWithInvalidResources() throws Exception { + public void testLoadFromValidJsonNodeWithInvalidResources() throws Exception { // Create JSON Object for testing - JSONObject jsonObject = new JSONObject("{\"name\": \"test\"}"); + Map testObj = createTestMap(); // Build resources - JSONObject res1 = new JSONObject("{\"name\": \"first-resource\", \"path\": [\"foo.txt\", \"bar.txt\", \"baz.txt\"]}"); - JSONObject res2 = new JSONObject("{\"name\": \"second-resource\", \"path\": [\"bar.txt\", \"baz.txt\"]}"); + JsonNode res1 = createNode("{\"name\": \"first-resource\", \"path\": [\"foo.txt\", \"bar.txt\", \"baz.txt\"]}"); + JsonNode res2 = createNode("{\"name\": \"second-resource\", \"path\": [\"bar.txt\", \"baz.txt\"]}"); - List resourceArrayList = new ArrayList<>(); + List resourceArrayList = new ArrayList<>(); resourceArrayList.add(res1); resourceArrayList.add(res2); - JSONArray resources = new JSONArray(resourceArrayList); - // Add the resources - jsonObject.put("resources", resources); + testObj.put("resources", resourceArrayList); // Build the datapackage - Package dp = new Package(jsonObject.toString(), getBasePath(), true); + Package dp = new Package(asString(testObj), getBasePath(), false); // Resolve the Resources -> FileNotFoundException due to non-existing files - exception.expect(FileNotFoundException.class); - List
tables = dp.getResource("first-resource").getTables(); + assertThrows(FileNotFoundException.class, () -> dp.getResource("first-resource").getTables()); } @Test - public void testLoadInvalidJsonObject() throws Exception { + public void testLoadInvalidJsonNode() throws Exception { // Create JSON Object for testing - JSONObject jsonObject = new JSONObject("{\"name\": \"test\"}"); + Map testObj = createTestMap(); // Build the datapackage, it will throw ValidationException because there are no resources. - exception.expect(DataPackageException.class); - Package dp = new Package(jsonObject.toString(), getBasePath(), true); + DataPackageValidationException ex = assertThrows( + DataPackageValidationException.class, + () -> new Package(asString(testObj), getBasePath(), true)); + Assertions.assertEquals("Trying to create a DataPackage from JSON, but no resource entries found", ex.getMessage()); } @Test - public void testLoadInvalidJsonObjectNoStrictValidation() throws Exception { + public void testLoadInvalidJsonNodeNoStrictValidation() throws Exception { // Create JSON Object for testing - JSONObject jsonObject = new JSONObject("{\"name\": \"test\"}"); + Map testObj = createTestMap(); // Build the datapackage, no strict validation by default - Package dp = new Package(jsonObject.toString(), getBasePath(), false); + Package dp = new Package(asString(testObj), getBasePath(), false); // Assert - Assert.assertNotNull(dp); + Assertions.assertNotNull(dp); } @Test public void testLoadFromFileWhenPathDoesNotExist() throws Exception { - exception.expect(DataPackageFileOrUrlNotFoundException.class); - new Package(new File("/this/path/does/not/exist").toPath(), true); + DataPackageFileOrUrlNotFoundException ex = assertThrows( + DataPackageFileOrUrlNotFoundException.class, + () -> new Package(new File("/this/path/does/not/exist").toPath(), true)); + } @Test @@ -164,10 +152,12 @@ public void testLoadFromFileWhenPathExists() throws Exception { // Build DataPackage instance based on source file path. Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); - // We're not asserting the String value since the order of the JSONObject elements is not guaranteed. + // We're not asserting the String value since the order of the JsonNode elements is not guaranteed. // Just compare the length of the String, should be enough. - JSONObject obj = new JSONObject(dp.getJson()); - Assert.assertEquals(obj.length(), new JSONObject(jsonString).length()); + JsonNode obj = createNode(dp.asJson()); + // a default 'profile' is being set, so the two packages will differ, unless a profile is added to the fixture data + Assertions.assertEquals(obj.get("resources").size(), createNode(jsonString).get("resources").size()); + Assertions.assertEquals(obj.get("name"), createNode(jsonString).get("name")); } @Test @@ -181,10 +171,10 @@ public void testLoadFromFileBasePath() throws Exception { // Build DataPackage instance based on source file path. Package dp = new Package(new File(basePath.toFile(), sourceFileName).toPath(), true); - Assert.assertNotNull(dp.getJson()); + Assertions.assertNotNull(dp.asJson()); // Check if base path was set properly; - Assert.assertEquals(basePath, dp.getBasePath()); + Assertions.assertEquals(basePath, dp.getBasePath()); } @@ -192,9 +182,9 @@ public void testLoadFromFileBasePath() throws Exception { public void testLoadFromFileWhenPathExistsButIsNotJson() throws Exception { // Get path of source file: String sourceFileAbsPath = PackageTest.class.getResource("/fixtures/not_a_json_datapackage.json").getPath(); - - exception.expect(JSONException.class); - Package dp = new Package(sourceFileAbsPath, getBasePath(), true); + DataPackageException ex = assertThrows( + DataPackageException.class, + () -> new Package(sourceFileAbsPath, getBasePath(), true)); } @@ -204,7 +194,7 @@ public void testValidUrl() throws Exception { // But could not resolve AbstractMethodError: https://stackoverflow.com/a/32696152/4030804 Package dp = new Package(validUrl, true); - Assert.assertNotNull(dp.getJson()); + Assertions.assertNotNull(dp.asJson()); } @Test @@ -213,8 +203,9 @@ public void testValidUrlWithInvalidJson() throws Exception { // But could not resolve AbstractMethodError: https://stackoverflow.com/a/32696152/4030804 URL url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/simple_invalid_datapackage.json"); - exception.expect(DataPackageException.class); - Package dp = new Package(url, true); + DataPackageException ex = assertThrows( + DataPackageException.class, + () -> new Package(url, true)); } @@ -224,7 +215,7 @@ public void testValidUrlWithInvalidJsonNoStrictValidation() throws Exception { "/master/src/test/resources/fixtures/simple_invalid_datapackage.json"); Package dp = new Package(url, false); - Assert.assertNotNull(dp.getJson()); + Assertions.assertNotNull(dp.asJson()); } @Test @@ -233,17 +224,20 @@ public void testUrlDoesNotExist() throws Exception { // But could not resolve AbstractMethodError: https://stackoverflow.com/a/32696152/4030804 URL url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/NON-EXISTANT-FOLDER/multi_data_datapackage.json"); - exception.expect(DataPackageException.class); - Package dp = new Package(url, true); + DataPackageException ex = assertThrows( + DataPackageException.class, + () -> new Package(url, true)); } @Test public void testLoadFromJsonFileResourceWithStrictValidationForInvalidNullPath() throws Exception { URL url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/invalid_multi_data_datapackage.json"); - - exception.expectMessage("Invalid Resource. The path property or the data and format properties cannot be null."); - Package dp = new Package(url, true); + + DataPackageValidationException ex = assertThrows( + DataPackageValidationException.class, + () -> new Package(url, true)); + Assertions.assertEquals("Invalid Resource. The path property or the data and format properties cannot be null.", ex.getMessage()); } @Test @@ -252,15 +246,22 @@ public void testLoadFromJsonFileResourceWithoutStrictValidationForInvalidNullPat "/master/src/test/resources/fixtures/invalid_multi_data_datapackage.json"); Package dp = new Package(url, false); - Assert.assertEquals("Invalid Resource. The path property or the data and " + + Assertions.assertEquals("Invalid Resource. The path property or the data and " + "format properties cannot be null.", dp.getErrors().get(0).getMessage()); } @Test public void testCreatingResourceWithInvalidPathNullValue() throws Exception { - exception.expectMessage("Invalid Resource. " + - "The path property cannot be null for file-based Resources."); - FilebasedResource resource = new FilebasedResource("resource-name", (Collection)null, null); + DataPackageException ex = assertThrows( + DataPackageException.class, + () -> {FilebasedResource.fromSource( + "resource-name", + null, + null, + TableDataSource.getDefaultEncoding());}); + Assertions.assertEquals("Invalid Resource. " + + "The path property cannot be null for file-based Resources.", ex.getMessage()); + } @@ -268,7 +269,7 @@ public void testCreatingResourceWithInvalidPathNullValue() throws Exception { public void testGetResources() throws Exception { // Create simple multi DataPackage from Json String Package dp = this.getDataPackageFromFilePath(true); - Assert.assertEquals(5, dp.getResources().size()); + Assertions.assertEquals(5, dp.getResources().size()); } @Test @@ -276,7 +277,7 @@ public void testGetExistingResource() throws Exception { // Create simple multi DataPackage from Json String Package dp = this.getDataPackageFromFilePath(true); Resource resource = dp.getResource("third-resource"); - Assert.assertNotNull(resource); + Assertions.assertNotNull(resource); } @@ -289,19 +290,19 @@ public void testReadTabseparatedResource() throws Exception { Dialect dialect = new Dialect(); dialect.setDelimiter("\t"); resource.setDialect(dialect); - Assert.assertNotNull(resource); - Listdata = resource.read(false); - Assert.assertEquals( 6, data.size()); - Assert.assertEquals("libreville", data.get(0)[0]); - Assert.assertEquals("0.41,9.29", data.get(0)[1]); - Assert.assertEquals( "dakar", data.get(1)[0]); - Assert.assertEquals("14.71,-17.53", data.get(1)[1]); - Assert.assertEquals("ouagadougou", data.get(2)[0]); - Assert.assertEquals("12.35,-1.67", data.get(2)[1]); - Assert.assertEquals("barranquilla", data.get(3)[0]); - Assert.assertEquals("10.98,-74.88", data.get(3)[1]); - Assert.assertEquals("cuidad de guatemala", data.get(5)[0]); - Assert.assertEquals("14.62,-90.56", data.get(5)[1]); + Assertions.assertNotNull(resource); + Listdata = resource.getData(false, false, false, false); + Assertions.assertEquals( 6, data.size()); + Assertions.assertEquals("libreville", data.get(0)[0]); + Assertions.assertEquals("0.41,9.29", data.get(0)[1]); + Assertions.assertEquals( "dakar", data.get(1)[0]); + Assertions.assertEquals("14.71,-17.53", data.get(1)[1]); + Assertions.assertEquals("ouagadougou", data.get(2)[0]); + Assertions.assertEquals("12.35,-1.67", data.get(2)[1]); + Assertions.assertEquals("barranquilla", data.get(3)[0]); + Assertions.assertEquals("10.98,-74.88", data.get(3)[1]); + Assertions.assertEquals("cuidad de guatemala", data.get(5)[0]); + Assertions.assertEquals("14.62,-90.56", data.get(5)[1]); } @@ -311,19 +312,19 @@ public void testReadTabseparatedResourceAndDialect() throws Exception { Package dp = this.getDataPackageFromFilePath( "/fixtures/tab_separated_datapackage_with_dialect.json", true); Resource resource = dp.getResource("first-resource"); - Assert.assertNotNull(resource); - Listdata = resource.read(false); - Assert.assertEquals( 6, data.size()); - Assert.assertEquals("libreville", data.get(0)[0]); - Assert.assertEquals("0.41,9.29", data.get(0)[1]); - Assert.assertEquals( "dakar", data.get(1)[0]); - Assert.assertEquals("14.71,-17.53", data.get(1)[1]); - Assert.assertEquals("ouagadougou", data.get(2)[0]); - Assert.assertEquals("12.35,-1.67", data.get(2)[1]); - Assert.assertEquals("barranquilla", data.get(3)[0]); - Assert.assertEquals("10.98,-74.88", data.get(3)[1]); - Assert.assertEquals("cuidad de guatemala", data.get(5)[0]); - Assert.assertEquals("14.62,-90.56", data.get(5)[1]); + Assertions.assertNotNull(resource); + Listdata = resource.getData(false, false, false, false); + Assertions.assertEquals( 6, data.size()); + Assertions.assertEquals("libreville", data.get(0)[0]); + Assertions.assertEquals("0.41,9.29", data.get(0)[1]); + Assertions.assertEquals( "dakar", data.get(1)[0]); + Assertions.assertEquals("14.71,-17.53", data.get(1)[1]); + Assertions.assertEquals("ouagadougou", data.get(2)[0]); + Assertions.assertEquals("12.35,-1.67", data.get(2)[1]); + Assertions.assertEquals("barranquilla", data.get(3)[0]); + Assertions.assertEquals("10.98,-74.88", data.get(3)[1]); + Assertions.assertEquals("cuidad de guatemala", data.get(5)[0]); + Assertions.assertEquals("14.62,-90.56", data.get(5)[1]); } @@ -332,23 +333,23 @@ public void testGetNonExistingResource() throws Exception { // Create simple multi DataPackage from Json String Package dp = this.getDataPackageFromFilePath(true); Resource resource = dp.getResource("non-existing-resource"); - Assert.assertNull(resource); + Assertions.assertNull(resource); } @Test public void testRemoveResource() throws Exception { Package dp = this.getDataPackageFromFilePath(true); - Assert.assertEquals(5, dp.getResources().size()); + Assertions.assertEquals(5, dp.getResources().size()); dp.removeResource("second-resource"); - Assert.assertEquals(4, dp.getResources().size()); + Assertions.assertEquals(4, dp.getResources().size()); dp.removeResource("third-resource"); - Assert.assertEquals(3, dp.getResources().size()); + Assertions.assertEquals(3, dp.getResources().size()); dp.removeResource("third-resource"); - Assert.assertEquals(3, dp.getResources().size()); + Assertions.assertEquals(3, dp.getResources().size()); } @Test @@ -357,38 +358,156 @@ public void testAddValidResource() throws Exception{ Package dp = this.getDataPackageFromFilePath(pathName,true); Path sourceFileAbsPath = Paths.get(PackageTest.class.getResource(pathName).toURI()); String basePath = sourceFileAbsPath.getParent().toString(); - Assert.assertEquals(5, dp.getResources().size()); + Assertions.assertEquals(5, dp.getResources().size()); List files = new ArrayList<>(); for (String s : Arrays.asList("cities.csv", "cities2.csv")) { files.add(new File(s)); } - Resource resource = Resource.build("new-resource", files, basePath); - Assert.assertTrue(resource instanceof FilebasedResource); - dp.addResource((FilebasedResource)resource); - Assert.assertEquals(6, dp.getResources().size()); + Resource resource = Resource.build("new-resource", files, basePath, TableDataSource.getDefaultEncoding()); + Assertions.assertTrue(resource instanceof FilebasedResource); + dp.addResource(resource); + Assertions.assertEquals(6, dp.getResources().size()); Resource gotResource = dp.getResource("new-resource"); - Assert.assertNotNull(gotResource); + Assertions.assertNotNull(gotResource); } - + @Test - public void testAddInvalidJSONResourceWithStrictValidation() throws Exception { - Package dp = this.getDataPackageFromFilePath(true); - Resource res = new JSONDataResource((String)null, testResources.toString()); + @DisplayName("Test getting resource data from a non-tabular datapackage, file based") + public void testNonTabularPackage() throws Exception{ + String pathName = "/fixtures/datapackages/non-tabular"; + Path resourcePath = TestUtil.getResourcePath(pathName); + Package dp = new Package(resourcePath, true); + + Resource resource = dp.getResource("logo-svg"); + Assertions.assertInstanceOf(FilebasedResource.class, resource); + byte[] rawData = (byte[])resource.getRawData(); + String s = new String (rawData).replaceAll("[\n\r]+", "\n"); + + byte[] testData = TestUtil.getResourceContent("/fixtures/files/frictionless-color-full-logo.svg"); + String t = new String (testData).replaceAll("[\n\r]+", "\n"); + Assertions.assertEquals(t, s); + } + + @Test + @DisplayName("Test getting resource data from a non-tabular datapackage, ZIP based") + public void testNonTabularPackageFromZip() throws Exception{ + String pathName = "/fixtures/zip/non-tabular.zip"; + Path resourcePath = TestUtil.getResourcePath(pathName); + Package dp = new Package(resourcePath, true); + + Resource resource = dp.getResource("logo-svg"); + Assertions.assertInstanceOf(FilebasedResource.class, resource); + byte[] rawData = (byte[])resource.getRawData(); + String s = new String (rawData).replaceAll("[\n\r]+", "\n"); + + byte[] testData = TestUtil.getResourceContent("/fixtures/files/frictionless-color-full-logo.svg"); + String t = new String (testData).replaceAll("[\n\r]+", "\n"); + Assertions.assertEquals(t, s); + } + + + @Test + @DisplayName("Test getting resource data from a non-tabular datapackage, URL based") + public void testNonTabularPackageUrl() throws Exception{ + URL input = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master" + + "/src/test/resources/fixtures/datapackages/non-tabular/datapackage.json"); + + Package dp = new Package(input, true); + + Resource resource = dp.getResource("logo-svg"); + Assertions.assertInstanceOf(URLbasedResource.class, resource); + byte[] rawData = (byte[])resource.getRawData(); + String s = new String (rawData).replaceAll("[\n\r]+", "\n"); - exception.expectMessage("Invalid Resource, it does not have a name property."); - dp.addResource(res); + byte[] testData = TestUtil.getResourceContent("/fixtures/files/frictionless-color-full-logo.svg"); + String t = new String (testData).replaceAll("[\n\r]+", "\n"); + Assertions.assertEquals(t, s); } @Test - public void testAddInvalidJSONResourceWithoutStrictValidation() throws Exception { - Package dp = this.getDataPackageFromFilePath(false); - Resource res = new JSONDataResource((String)null, testResources.toString()); - dp.addResource(res); + @DisplayName("Test setting the 'profile' property") + public void testSetProfile() throws Exception { + Path tempDirPath = Files.createTempDirectory("datapackage-"); + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + Package dp = new Package(resourcePath, true); + + Assertions.assertEquals(Profile.PROFILE_TABULAR_DATA_PACKAGE, dp.getProfile()); + + dp.setProfile(PROFILE_DATA_PACKAGE_DEFAULT); + Assertions.assertEquals(Profile.PROFILE_DATA_PACKAGE_DEFAULT, dp.getProfile()); + + File outFile = new File (tempDirPath.toFile(), "datapackage.json"); + dp.writeJson(outFile); + String content = String.join("\n", Files.readAllLines(outFile.toPath())); + JsonNode jsonNode = JsonUtil.getInstance().readValue(content); + String profile = jsonNode.get("profile").asString(); + Assertions.assertEquals(Profile.PROFILE_DATA_PACKAGE_DEFAULT, profile); + Assertions.assertEquals(Profile.PROFILE_DATA_PACKAGE_DEFAULT, dp.getProfile()); + } + + @Test + @DisplayName("Test setting the 'image' propertye") + public void testSetImage() throws Exception { + Path tempDirPath = Files.createTempDirectory("datapackage-"); + String fName = "/fixtures/datapackages/with-image"; + Path resourcePath = TestUtil.getResourcePath(fName); + Package dp = new Package(resourcePath, true); + + byte[] imageData = TestUtil.getResourceContent("/fixtures/files/test.png"); + dp.setImage("test.png", imageData); + + File tmpFile = new File(tempDirPath.toFile(), "saved-pkg.zip"); + dp.write(tmpFile, true); + + // Read the datapckage we just saved in the temp dir. + Package readPackage = new Package(tmpFile.toPath(), false); + byte[] readImageData = readPackage.getImage(); + Assertions.assertArrayEquals(imageData, readImageData); + } + + @Test + @DisplayName("Test setting the 'image' property, ZIP file") + public void testSetImageZip() throws Exception { + Path tempDirPath = Files.createTempDirectory("datapackage-"); + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + Package dp = new Package(resourcePath, true); + + byte[] imageData = TestUtil.getResourceContent("/fixtures/files/test.png"); + dp.setImage("test.png", imageData); + + File tmpFile = new File(tempDirPath.toFile(), "saved-pkg.zip"); + dp.write(tmpFile, true); + + // Read the datapckage we just saved in the zip file. + Package readPackage = new Package(tmpFile.toPath(), false); + byte[] readImageData = readPackage.getImage(); + Assertions.assertArrayEquals(imageData, readImageData); + } + + @Test + @DisplayName("Test setting invalid 'profile' property, must throw") + public void testSetInvalidProfile() throws Exception { + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + Package dp = new Package(resourcePath, true); + + Assertions.assertThrows(DataPackageValidationException.class, + () -> dp.setProfile(PROFILE_DATA_RESOURCE_DEFAULT)); + Assertions.assertThrows(DataPackageValidationException.class, + () -> dp.setProfile(PROFILE_TABULAR_DATA_RESOURCE)); + } - Assert.assertTrue( dp.getErrors().size() > 0); - Assert.assertEquals("Invalid Resource, it does not have a name property.", dp.getErrors().get(0).getMessage()); + @Test + public void testCreateInvalidJSONResource() throws Exception { + Package dp = this.getDataPackageFromFilePath(true); + DataPackageException dpe = assertThrows(DataPackageException.class, + () -> {Resource res = new JSONDataResource(null, testResources); + dp.addResource(res);}); + Assertions.assertEquals("Invalid Resource, it does not have a name property.", dpe.getMessage()); } @@ -404,11 +523,11 @@ public void testAddDuplicateNameResourceWithStrictValidation() throws Exception for (String s : Arrays.asList("cities.csv", "cities2.csv")) { files.add(new File(s)); } - Resource resource = Resource.build("third-resource", files, basePath); - Assert.assertTrue(resource instanceof FilebasedResource); - - exception.expectMessage("A resource with the same name already exists."); - dp.addResource((FilebasedResource)resource); + Resource resource = Resource.build("third-resource", files, basePath, TableDataSource.getDefaultEncoding()); + Assertions.assertInstanceOf(FilebasedResource.class, resource); + + DataPackageException dpe = assertThrows(DataPackageException.class, () -> dp.addResource(resource)); + Assertions.assertEquals("A resource with the same name already exists.", dpe.getMessage()); } @Test @@ -422,12 +541,12 @@ public void testAddDuplicateNameResourceWithoutStrictValidation() throws Excepti for (String s : Arrays.asList("cities.csv", "cities2.csv")) { files.add(new File(s)); } - Resource resource = Resource.build("third-resource", files, basePath); - Assert.assertTrue(resource instanceof FilebasedResource); - dp.addResource((FilebasedResource)resource); + Resource resource = Resource.build("third-resource", files, basePath, TableDataSource.getDefaultEncoding()); + Assertions.assertInstanceOf(FilebasedResource.class, resource); + dp.addResource(resource); - Assert.assertEquals(1, dp.getErrors().size()); - Assert.assertEquals("A resource with the same name already exists.", dp.getErrors().get(0).getMessage()); + Assertions.assertEquals(1, dp.getErrors().size()); + Assertions.assertEquals("A resource with the same name already exists.", dp.getErrors().get(0).getMessage()); } @@ -439,16 +558,15 @@ public void testSaveToJsonFile() throws Exception{ savedPackage.write(tempDirPath.toFile(), false); Package readPackage = new Package(tempDirPath.resolve(Package.DATAPACKAGE_FILENAME),false); - JSONObject readPackageJson = new JSONObject(readPackage.getJson()) ; - JSONObject savedPackageJson = new JSONObject(savedPackage.getJson()) ; - Assert.assertTrue(readPackageJson.similar(savedPackageJson)); + JsonNode readPackageJson = createNode(readPackage.asJson()) ; + JsonNode savedPackageJson = createNode(savedPackage.asJson()) ; + Assertions.assertEquals(readPackageJson, savedPackageJson); } @Test public void testSaveToAndReadFromZipFile() throws Exception{ Path tempDirPath = Files.createTempDirectory("datapackage-"); File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); - System.out.println(createdFile); // saveDescriptor the datapackage in zip file. Package originalPackage = this.getDataPackageFromFilePath(true); @@ -458,9 +576,9 @@ public void testSaveToAndReadFromZipFile() throws Exception{ Package readPackage = new Package(createdFile.toPath(), false); // Check if two data packages are have the same key/value pairs. - // For some reason JSONObject.similar() is not working even though both - // json objects are exactly the same. Just compare lengths then. - Assert.assertEquals(readPackage.getJson().toString().length(), originalPackage.getJson().toString().length()); + String expected = readPackage.asJson(); + String actual = originalPackage.asJson(); + Assertions.assertEquals(expected, actual); } @@ -475,45 +593,145 @@ public void testReadFromZipFileWithDirectoryHierarchy() throws Exception{ Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); Resource r = dp.getResource("currencies"); - List data = r.read(false); - Assert.assertEquals(2, data.size()); - Assert.assertArrayEquals(usdTestData, data.get(0)); - Assert.assertArrayEquals(gbpTestData, data.get(1)); + List data = r.getData(false, false, false, false); + Assertions.assertEquals(2, data.size()); + Assertions.assertArrayEquals(usdTestData, data.get(0)); + Assertions.assertArrayEquals(gbpTestData, data.get(1)); } - + + /* + * ensure the zip file is closed after reading so we don't leave file handles dangling. + */ + @Test + public void testClosesZipFile() throws Exception{ + Path tempDirPath = Files.createTempDirectory("datapackage-"); + File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); + Path resourcePath = TestUtil.getResourcePath("/fixtures/zip/countries-and-currencies.zip"); + Files.copy(resourcePath, createdFile.toPath()); + + Package dp = new Package(createdFile.toPath(), true); + Resource r = dp.getResource("currencies"); + createdFile.delete(); + Assertions.assertFalse(createdFile.exists()); + } + + // Archive file name doesn't end with ".zip" + @Test + @DisplayName("Read package from a ZIP file with different suffix") + public void testReadFromZipFileWithDifferentSuffix() throws Exception{ + String[] usdTestData = new String[]{"USD", "US Dollar", "$"}; + String[] gbpTestData = new String[]{"GBP", "Pound Sterling", "£"}; + String sourceFileAbsPath = ResourceTest.class.getResource("/fixtures/zip/countries-and-currencies.zap").getPath(); + + Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); + Resource r = dp.getResource("currencies"); + + List data = r.getData(false, false, false, false); + Assertions.assertEquals(2, data.size()); + Assertions.assertArrayEquals(usdTestData, data.get(0)); + Assertions.assertArrayEquals(gbpTestData, data.get(1)); + } + @Test + @DisplayName("Datapackage with invalid name for descriptor (ie. not 'datapackage.java', must throw") public void testReadFromZipFileWithInvalidDatapackageFilenameInside() throws Exception{ String sourceFileAbsPath = PackageTest.class.getResource("/fixtures/zip/invalid_filename_datapackage.zip").getPath(); - - exception.expect(DataPackageException.class); - new Package(new File(sourceFileAbsPath).toPath(), false); + + DataPackageException dpe = assertThrows(DataPackageException.class, + () -> new Package(new File(sourceFileAbsPath).toPath(), false)); + Assertions.assertEquals("The zip file does not contain the expected file: datapackage.json", dpe.getMessage()); } @Test + @DisplayName("Read package from a ZIP with invalid descriptor, must throw") public void testReadFromZipFileWithInvalidDatapackageDescriptorAndStrictValidation() throws Exception{ Path sourceFileAbsPath = Paths .get(PackageTest.class.getResource("/fixtures/zip/invalid_datapackage.zip").toURI()); - - exception.expect(DataPackageException.class); - new Package(sourceFileAbsPath.toFile().toPath(), true); + + assertThrows(DataPackageException.class, + () -> new Package(sourceFileAbsPath.toFile().toPath(), true)); } @Test + @DisplayName("Read package from a non-existing path, must throw") public void testReadFromInvalidZipFilePath() throws Exception{ - exception.expect(IOException.class); File invalidFile = new File ("/invalid/path/does/not/exist/datapackage.zip"); - Package p = new Package(invalidFile.toPath(), false); + assertThrows(DataPackageFileOrUrlNotFoundException.class, + () -> new Package(invalidFile.toPath(), false)); + } + + @Test + @DisplayName("Write datapackage with an image to a folder") + public void testWriteImageToFolderPackage() throws Exception{ + File dataDirectory = TestUtil.getTestDataDirectory(); + Package pkg = new Package(new File( getBasePath().toFile(), "datapackages/employees/datapackage.json").toPath(), false); + File imgFile = new File (dataDirectory, "fixtures/files/frictionless-color-full-logo.svg"); + byte [] fileData = Files.readAllBytes(imgFile.toPath()); + Path tempDirPath = Files.createTempDirectory("datapackage-"); + + pkg.setImage("logo/ file.svg", fileData); + File dir = new File (tempDirPath.toFile(), "with-image"); + Path dirPath = Files.createDirectory(dir.toPath(), new FileAttribute[] {}); + pkg.write(dirPath.toFile(), false); + if (verbose) { + System.out.println(tempDirPath); + } + File descriptor = new File (dir, "datapackage.json"); + String json = String.join("\n", Files.readAllLines(descriptor.toPath())); + Assertions.assertFalse(json.contains("\"imageData\"")); + } + + @Test + @DisplayName("Write datapackage with an image to a ZIP file") + public void testWriteImageToZipPackage() throws Exception{ + File dataDirectory = TestUtil.getTestDataDirectory(); + File imgFile = new File (dataDirectory, "fixtures/files/frictionless-color-full-logo.svg"); + byte [] fileData = Files.readAllBytes(imgFile.toPath()); + Path tempDirPath = Files.createTempDirectory("datapackage-"); + Path resourcePath = TestUtil.getResourcePath("/fixtures/zip/countries-and-currencies.zip"); + File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); + Files.copy(resourcePath, createdFile.toPath()); + + Package dp = new Package(createdFile.toPath(), true); + dp.setImage("logo/ file.svg", fileData); + dp.write(new File(tempDirPath.toFile(), "with-image.zip"), true); + if (verbose) { + System.out.println(tempDirPath); + } + + File withImageFile = new File(tempDirPath.toFile(), "with-image.zip"); + Package withImageDp = new Package(withImageFile.toPath(), true); + byte[] readImageData = withImageDp.getImage(); + Assertions.assertArrayEquals(fileData, readImageData); } + + @Test + @DisplayName("Write datapackage using a Consumer function to fingerprint files") + public void testWriteWithConsumer() throws Exception{ + File refDescriptor = new File(getBasePath().toFile(), "datapackages/employees/datapackage.json"); + Package pkg = new Package(refDescriptor.toPath(), false); + Path tempDirPath = Files.createTempDirectory("datapackage-"); + + File dir = new File (tempDirPath.toFile(), "test-package"); + Path dirPath = Files.createDirectory(dir.toPath()); + pkg.write(dirPath.toFile(), PackageTest::fingerprintFiles, false); + if (verbose) { + System.out.println(tempDirPath); + } + File fingerprints = new File (dir, "fingerprints.txt"); + String content = String.join("\n", Files.readAllLines(fingerprints.toPath())); + String refContent = + "datapackage.json\te3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n" + + "schema.json\te3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + Assertions.assertEquals(refContent, content); + } @Test public void testMultiPathIterationForLocalFiles() throws Exception{ Package pkg = this.getDataPackageFromFilePath(true); Resource resource = pkg.getResource("first-resource"); - - // Set the profile to tabular data resource. - resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); - + // Expected data. List expectedData = this.getAllCityData(); @@ -527,8 +745,8 @@ public void testMultiPathIterationForLocalFiles() throws Exception{ String city = record[0]; String location = record[1]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], location); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], location); expectedDataIndex++; } @@ -555,8 +773,8 @@ public void testMultiPathIterationForRemoteFile() throws Exception{ String city = record[0]; String location = record[1]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], location); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], location); expectedDataIndex++; } @@ -571,13 +789,13 @@ public void testResourceSchemaDereferencingForLocalDataFileAndRemoteSchemaFile() String schemaJsonString =getFileContents("/fixtures/schema/population_schema.json"); Schema expectedSchema = Schema.fromJson(schemaJsonString, true); - Assert.assertEquals(expectedSchema, resource.getSchema()); + Assertions.assertEquals(expectedSchema, resource.getSchema()); // Get JSON Object - JSONObject expectedSchemaJson = new JSONObject(expectedSchema.getJson()); - JSONObject testSchemaJson = new JSONObject(resource.getSchema().getJson()); + JsonNode expectedSchemaJson = createNode(expectedSchema.asJson()); + JsonNode testSchemaJson = createNode(resource.getSchema().asJson()); // Compare JSON objects - Assert.assertTrue("Schemas don't match", expectedSchemaJson.similar(testSchemaJson)); + Assertions.assertEquals(expectedSchemaJson, testSchemaJson, "Schemas don't match"); } @Test @@ -589,21 +807,74 @@ public void testResourceSchemaDereferencingForRemoteDataFileAndLocalSchemaFile() String schemaJsonString =getFileContents("/fixtures/schema/population_schema.json"); Schema expectedSchema = Schema.fromJson(schemaJsonString, true); - Assert.assertEquals(expectedSchema, resource.getSchema()); + Assertions.assertEquals(expectedSchema, resource.getSchema()); // Get JSON Object - JSONObject expectedSchemaJson = new JSONObject(expectedSchema.getJson()); - JSONObject testSchemaJson = new JSONObject(resource.getSchema().getJson()); + JsonNode expectedSchemaJson = createNode(expectedSchema.asJson()); + JsonNode testSchemaJson = createNode(resource.getSchema().asJson()); // Compare JSON objects - Assert.assertTrue("Schemas don't match", expectedSchemaJson.similar(testSchemaJson)); + Assertions.assertEquals(expectedSchemaJson, testSchemaJson, "Schemas don't match"); } - - /** TODO: Implement more thorough testing. + + @Test + public void testAddPackageProperty() throws Exception{ + Object[] entries = new Object[]{"K", 3.2, 2}; + Map props = new LinkedHashMap<>(); + props.put("time", "s"); + props.put("length", 3.2); + props.put("count", 7); + + Path pkgFile = TestUtil.getResourcePath("/fixtures/datapackages/employees/datapackage.json"); + Package p = new Package(pkgFile, true); + + p.setProperty("mass unit", "kg"); + p.setProperty("mass flow", 3.2); + p.setProperty("number of parcels", 5); + p.setProperty("entries", entries); + p.setProperty("props", props); + p.setProperty("null", null); + Assertions.assertEquals("kg", p.getProperty("mass unit"), "JSON doesn't match"); + Assertions.assertEquals(new BigDecimal("3.2"), p.getProperty("mass flow"), "JSON doesn't match"); + Assertions.assertEquals(new BigInteger("5"), p.getProperty("number of parcels"), "JSON doesn't match"); + Assertions.assertEquals(Arrays.asList(entries), p.getProperty("entries"), "JSON doesn't match"); + Assertions.assertEquals(props, p.getProperty("props"), "JSON doesn't match"); + Assertions.assertNull(p.getProperty("null")); + } + @Test - public void testResourceSchemaDereferencingWithInvalidResourceSchema() throws DataPackageException, IOException{ - exception.expect(ValidationException.class); - Package pkg = this.getDataPackageFromFilePath(true, "/fixtures/multi_data_datapackage_with_invalid_resource_schema.json"); - }**/ + // tests the setProperties() method of Package + public void testSetPackageProperties() throws Exception{ + Object[] entries = new Object[]{"K", 3.2, 2}; + Map map = new LinkedHashMap<>(); + Map props = new LinkedHashMap<>(); + props.put("time", "s"); + props.put("length", 3.2); + props.put("count", 7); + map.put("mass unit", "kg"); + map.put("mass flow", 3.2); + map.put("number of parcels", 5); + map.put("entries", entries); + map.put("props", props); + map.put("null", null); + + Path pkgFile = TestUtil.getResourcePath("/fixtures/datapackages/employees/datapackage.json"); + Package p = new Package(pkgFile, false); + p.setProperties(map); + Assertions.assertEquals("kg", p.getProperty("mass unit"), "JSON doesn't match"); + Assertions.assertEquals(new BigDecimal("3.2"), p.getProperty("mass flow"), "JSON doesn't match"); + Assertions.assertEquals( new BigInteger("5"), p.getProperty("number of parcels"), "JSON doesn't match"); + Assertions.assertEquals(Arrays.asList(entries), p.getProperty("entries"), "JSON doesn't match"); + Assertions.assertEquals(props, p.getProperty("props"), "JSON doesn't match"); + Assertions.assertNull(p.getProperty("null"), "JSON doesn't match"); + } + + // test schema validation. Schema is invalid, must throw + @Test + public void testResourceSchemaDereferencingWithInvalidResourceSchema() { + assertThrows(ValidationException.class, () -> this.getDataPackageFromFilePath( + "/fixtures/multi_data_datapackage_with_invalid_resource_schema.json", true + )); + } @Test public void testResourceDialectDereferencing() throws Exception { @@ -615,7 +886,7 @@ public void testResourceDialectDereferencing() throws Exception { String dialectJsonString =getFileContents("/fixtures/dialect.json"); // Compare. - Assert.assertEquals(Dialect.fromJson(dialectJsonString), resource.getDialect()); + Assertions.assertEquals(Dialect.fromJson(dialectJsonString), resource.getDialect()); } @Test @@ -624,42 +895,210 @@ public void testAdditionalProperties() throws Exception { String sourceFileAbsPath = PackageTest.class.getResource(fName).getPath(); Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); - Object creator = dp.getOtherProperty("creator"); - Assert.assertNotNull(creator); - Assert.assertEquals(String.class, creator.getClass()); - Assert.assertEquals("Horst", creator); + Object creator = dp.getProperty("creator"); + Assertions.assertNotNull(creator); + Assertions.assertEquals("Horst", creator); - Object testprop = dp.getOtherProperty("testprop"); - Assert.assertNotNull(testprop); - Assert.assertTrue(testprop instanceof JSONObject); + Object testprop = dp.getProperty("testprop"); + Assertions.assertNotNull(testprop); + Assertions.assertTrue(testprop instanceof Map); - Object testarray = dp.getOtherProperty("testarray"); - Assert.assertNotNull(testarray); - Assert.assertTrue(testarray instanceof JSONArray); + Object testarray = dp.getProperty("testarray"); + Assertions.assertNotNull(testarray); + Assertions.assertTrue(testarray instanceof ArrayList); - Object resObj = dp.getOtherProperty("resources"); - Assert.assertNull(resObj); + Object resObj = dp.getProperty("something"); + Assertions.assertNull(resObj); } @Test public void testBeanResource1() throws Exception { - Package pkg = new Package(new File( getBasePath().toFile(), "datapackages/employees/datapackage.json").toPath(), true); + Package pkg = new Package(new File(getBasePath().toFile(), "datapackages/bean-iterator/datapackage.json").toPath(), true); Resource resource = pkg.getResource("employee-data"); - final List employees = resource.read(EmployeeBean.class); - Assert.assertEquals(3, employees.size()); + final List employees = resource.getData(EmployeeBean.class); + Assertions.assertEquals(3, employees.size()); EmployeeBean frank = employees.get(1); - Assert.assertEquals("Frank McKrank", frank.getName()); - Assert.assertEquals("1992-02-14", new DateField("date").formatValue(frank.getDateOfBirth(), null, null)); - Assert.assertFalse(frank.getAdmin()); - Assert.assertEquals("(90.0, 45.0, NaN)", frank.getAddressCoordinates().toString()); - Assert.assertEquals("PT15M", frank.getContractLength().toString()); + Assertions.assertEquals("Frank McKrank", frank.getName()); + Assertions.assertEquals("1992-02-14", new DateField("date").formatValueAsString(frank.getDateOfBirth(), null, null)); + Assertions.assertFalse(frank.getAdmin()); + Assertions.assertEquals("(90.0, 45.0, NaN)", frank.getAddressCoordinates().toString()); + Assertions.assertEquals("PT15M", frank.getContractLength().toString()); Map info = frank.getInfo(); Assertions.assertEquals(45, info.get("pin")); Assertions.assertEquals(83.23, info.get("rate")); Assertions.assertEquals(90, info.get("ssn")); } + @Test + @DisplayName("Test read a Package with all fields defined in https://specs.frictionlessdata.io/data-package/#metadata") + public void testReadPackageAllFields() throws Exception{ + Path pkgFile = TestUtil.getResourcePath("/fixtures/full_spec_datapackage.json"); + Package p = new Package(pkgFile, false); + Assertions.assertEquals( "9e2429be-a43e-4051-aab5-981eb27fe2e8", p.getId()); + Assertions.assertEquals( "world-full", p.getName()); + Assertions.assertEquals( "world population data", p.getTitle()); + Assertions.assertEquals( "tabular-data-package", p.getProfile()); + Assertions.assertEquals("A datapackage for world population data, featuring all fields from https://specs.frictionlessdata.io/data-package/#language", p.getDescription()); + Assertions.assertEquals("1.0.1", p.getVersion()); + Assertions.assertEquals( "https://example.com/world-population-data", p.getHomepage().toString()); + Assertions.assertEquals(1, p.getLicenses().size()); + Assertions.assertEquals(1, p.getSources().size()); + Assertions.assertEquals(2, p.getContributors().size()); + Assertions.assertArrayEquals(new String[] {"world", "population", "world bank"}, p.getKeywords().toArray()); + Assertions.assertEquals( "https://github.com/frictionlessdata/datapackage-java/tree/main/src/test/resources/fixtures/datapackages/with-image/test.png", p.getImagePath()); + Assertions.assertEquals(ZonedDateTime.parse("1985-04-12T23:20:50.52Z"), p.getCreated()); + Assertions.assertEquals(1, p.getResources().size()); + + License license = p.getLicenses().get(0); + Assertions.assertEquals("ODC-PDDL-1.0", license.getName()); + Assertions.assertEquals("http://opendatacommons.org/licenses/pddl/", license.getPath()); + Assertions.assertEquals("Open Data Commons Public Domain Dedication and License v1.0", license.getTitle()); + + Source source = p.getSources().get(0); + Assertions.assertEquals("http://data.worldbank.org/indicator/NY.GDP.MKTP.CD", source.getPath()); + Assertions.assertEquals("World Bank and OECD", source.getTitle()); + + Contributor c = p.getContributors().get(1); + Assertions.assertEquals("Jim Beam", c.getTitle()); + Assertions.assertEquals("jim@example.com", c.getEmail()); + Assertions.assertEquals("https://www.example.com", c.getPath().toString()); + Assertions.assertEquals("wrangler", c.getRole()); + Assertions.assertEquals("Example Corp", c.getOrganization()); + } + + @Test + // check for https://github.com/frictionlessdata/datapackage-java/issues/46 + @DisplayName("Show that minimum constraints work") + void validateDataPackage() throws Exception { + Package dp = this.getDataPackageFromFilePath( + "/fixtures/datapackages/constraint-violation/datapackage.json", true); + Resource resource = dp.getResource("person_data"); + ConstraintsException exception = assertThrows(ConstraintsException.class, () -> resource.getData(false, false, true, false)); + + // Assert the validation messages + Assertions.assertNotNull(exception.getMessage()); + Assertions.assertFalse(exception.getMessage().isEmpty()); + } + + @Test + @DisplayName("Datapackage with same data in different formats, lenient validation") + void validateDataPackageDifferentFormats() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/different-data-formats_incl_invalid/datapackage.json"); + Package dp = new Package(resourcePath, false); + List teamsWithHeaders = dp.getResource("teams_with_headers_csv_file_with_schema").getData(false, false, true, false); + List teamsWithHeadersCsvFileNoSchema = dp.getResource("teams_with_headers_csv_file_no_schema").getData(false, false, true, false); + List teamsNoHeadersCsvFileNoSchema = dp.getResource("teams_no_headers_csv_file_no_schema").getData(false, false, true, false); + List teamsNoHeadersCsvInlineNoSchema = dp.getResource("teams_no_headers_inline_csv_no_schema").getData(false, false, true, false); + + List teamsArraysInline = dp.getResource("teams_arrays_inline_with_headers_with_schema").getData(false, false, true, false); + List teamsObjectsInline = dp.getResource("teams_objects_inline_with_schema").getData(false, false, true, false); + List teamsArrays = dp.getResource("teams_arrays_file_with_headers_with_schema").getData(false, false, true, false); + List teamsObjects = dp.getResource("teams_objects_file_with_schema").getData(false, false, true, false); + List teamsArraysInlineNoSchema = dp.getResource("teams_arrays_inline_with_headers_no_schema").getData(false, false, true, false); + + // ensure tables without headers throw errors on reading if a Schema is set + TableValidationException ex = assertThrows(TableValidationException.class, + () -> dp.getResource("teams_arrays_no_headers_inline_with_schema").getData(false, false, true, false)); + Assertions.assertEquals("Field 'id' not found in table headers or table has no headers.", ex.getMessage()); + + TableValidationException ex2 = assertThrows(TableValidationException.class, + () -> dp.getResource("teams_no_headers_inline_csv_with_schema").getData(false, false, true, false)); + Assertions.assertEquals("Field 'id' not found in table headers or table has no headers.", ex2.getMessage()); + + TableValidationException ex3 = assertThrows(TableValidationException.class, + () -> dp.getResource("teams_no_headers_csv_file_with_schema").getData(false, false, true, false)); + Assertions.assertEquals("Field 'id' not found in table headers or table has no headers.", ex3.getMessage()); + + Assertions.assertArrayEquals(getFullTeamsData().toArray(), teamsWithHeaders.toArray()); + Assertions.assertArrayEquals(getFullTeamsData().toArray(), teamsArraysInline.toArray()); + Assertions.assertArrayEquals(getFullTeamsData().toArray(), teamsObjectsInline.toArray()); + Assertions.assertArrayEquals(getFullTeamsData().toArray(), teamsArrays.toArray()); + Assertions.assertArrayEquals(getFullTeamsData().toArray(), teamsObjects.toArray()); + + // those without Schema lose the type information. With header row means all data is there + Assertions.assertArrayEquals(getFullTeamsDataString().toArray(), teamsWithHeadersCsvFileNoSchema.toArray()); + Assertions.assertArrayEquals(getFullTeamsDataString().toArray(), teamsArraysInlineNoSchema.toArray()); + + // those without a header row and with no Schema will lose the first row of data (skipped as a header row). Seems wrong but that's what the python port does + Assertions.assertArrayEquals(getTeamsDataStringMissingFirstRow().toArray(), teamsNoHeadersCsvFileNoSchema.toArray()); + Assertions.assertArrayEquals(getTeamsDataStringMissingFirstRow().toArray(), teamsNoHeadersCsvInlineNoSchema.toArray()); + } + + @Test + @DisplayName("Datapackage with same data in different valid formats, strict validation") + void validateDataPackageDifferentFormatsStrict() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/different-valid-data-formats/datapackage.json"); + Package dp = new Package(resourcePath, true); + + List teamsWithHeaders = dp.getResource("teams_with_headers_csv_file").getData(false, false, true, false); + List teamsArraysInline = dp.getResource("teams_arrays_inline").getData(false, false, true, false); + List teamsObjectsInline = dp.getResource("teams_objects_inline").getData(false, false, true, false); + List teamsArrays = dp.getResource("teams_arrays_file").getData(false, false, true, false); + List teamsObjects = dp.getResource("teams_objects_file").getData(false, false, true, false); + + Assertions.assertArrayEquals(teamsWithHeaders.toArray(), getFullTeamsData().toArray()); + Assertions.assertArrayEquals(teamsArraysInline.toArray(), getFullTeamsData().toArray()); + Assertions.assertArrayEquals(teamsObjectsInline.toArray(), getFullTeamsData().toArray()); + Assertions.assertArrayEquals(teamsArrays.toArray(), getFullTeamsData().toArray()); + Assertions.assertArrayEquals(teamsObjects.toArray(), getFullTeamsData().toArray()); + } + + private static List getFullTeamsData() { + List expectedData = new ArrayList<>(); + expectedData.add(new Object[]{BigInteger.valueOf(1), "Arsenal", "London"}); + expectedData.add(new Object[]{BigInteger.valueOf(2), "Real", "Madrid"}); + expectedData.add(new Object[]{BigInteger.valueOf(3), "Bayern", "Munich"}); + return expectedData; + } + + private static List getFullTeamsDataString() { + List expectedData = new ArrayList<>(); + expectedData.add(new Object[]{"1", "Arsenal", "London"}); + expectedData.add(new Object[]{"2", "Real", "Madrid"}); + expectedData.add(new Object[]{"3", "Bayern", "Munich"}); + return expectedData; + } + + private static List getTeamsDataStringMissingFirstRow() { + List expectedData = new ArrayList<>(); + expectedData.add(new Object[]{"2", "Real", "Madrid"}); + expectedData.add(new Object[]{"3", "Bayern", "Munich"}); + return expectedData; + } + + + private static void fingerprintFiles(Path path) { + List fingerprints = new ArrayList<>(); + MessageDigest md; + try { + md = MessageDigest.getInstance("SHA-256"); + File[] files = path.toFile().listFiles(); + TreeSet sortedFiles = new TreeSet<>(Arrays.asList(files)); + for (File f : sortedFiles) { + if (f.isFile()) { + String content = String.join("\n", Files.readAllLines(f.toPath())); + content = content.replaceAll("[\\n\\r]+", "\n"); + md.digest(content.getBytes()); + + StringBuilder result = new StringBuilder(); + for (byte b : md.digest()) { + result.append(String.format("%02x", b)); + } + fingerprints.add(f.getName() + "\t" + result); + md.reset(); + } + } + + File outFile = new File(path.toFile(), "fingerprints.txt"); + try (FileWriter wr = new FileWriter(outFile)) { + wr.write(String.join("\n", fingerprints)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private Package getDataPackageFromFilePath(String datapackageFilePath, boolean strict) throws Exception { // Get string content version of source file. String jsonString = getFileContents(datapackageFilePath); @@ -676,18 +1115,14 @@ private Package getDataPackageFromFilePath(boolean strict) throws Exception { private static String getFileContents(String fileName) { try { - // Create file-URL of source file: - URL sourceFileUrl = PackageTest.class.getResource(fileName); - // Get path of URL - Path path = Paths.get(sourceFileUrl.toURI()); - return new String(Files.readAllBytes(path)); + return new String(TestUtil.getResourceContent(fileName)); } catch (Exception ex) { throw new RuntimeException(ex); } } private List getAllCityData(){ - List expectedData = new ArrayList(); + List expectedData = new ArrayList<>(); expectedData.add(new String[]{"libreville", "0.41,9.29"}); expectedData.add(new String[]{"dakar", "14.71,-17.53"}); expectedData.add(new String[]{"ouagadougou", "12.35,-1.67"}); @@ -701,6 +1136,20 @@ private List getAllCityData(){ return expectedData; } + private static JsonNode createNode(String json) { + return JsonUtil.getInstance().createNode(json); + } + + private Map createTestMap(){ + HashMap map = new HashMap<>(); + map.put("name", "test"); + return map; + } + + private String asString(Object object) { + return JsonUtil.getInstance().serialize(object); + } + //TODO: come up with attribute edit tests: // Examples here: https://github.com/frictionlessdata/datapackage-py/blob/master/tests/test_datapackage.py } diff --git a/src/test/java/io/frictionlessdata/datapackage/SpecificationValidityTest.java b/src/test/java/io/frictionlessdata/datapackage/SpecificationValidityTest.java deleted file mode 100644 index a6a18b9..0000000 --- a/src/test/java/io/frictionlessdata/datapackage/SpecificationValidityTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package io.frictionlessdata.datapackage; - -import org.everit.json.schema.ArraySchema; -import org.everit.json.schema.ObjectSchema; -import org.everit.json.schema.Schema; -import org.everit.json.schema.loader.SchemaLoader; -import org.json.JSONArray; -import org.json.JSONObject; -import org.json.JSONTokener; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; - -public class SpecificationValidityTest { - - private String testVal2 = "[" + - "{\"schema\":\"https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/schema/population_schema.json\"," + - "\"path\":[\"https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities.csv\"," + - " \"https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities2.csv\"," + - " \"https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities3.csv\"]," + - "\"name\":\"third-resource\"}," + - "]"; -/* - @Test - @DisplayName("Test that a schema can be defined via a URL") - // Test for https://github.com/frictionlessdata/specs/issues/645 - void testValidationURLAsSchemaReference() throws Exception{ - JSONArray jsonObjectToValidate = new JSONArray(testVal2); - InputStream inputStream = Validator.class.getResourceAsStream("/schemas/data-package.json"); - if(inputStream != null) { - JSONObject rawSchema = new JSONObject(new JSONTokener(inputStream)); - Schema objSchema = SchemaLoader.load(rawSchema); - Map schemas = ((ObjectSchema)objSchema).getPropertySchemas(); - ArraySchema schema = (ArraySchema)schemas.get("resources"); - schema.validate(jsonObjectToValidate); // throws a ValidationException if this object is invalid - } else { - throw new FileNotFoundException(); - } - } - - - @Test - void testLoadFromFileWhenPathExists() throws Exception { - String fName = "/testsuite-data/basic-csv/datapackage.json"; - // Get string content version of source file. - String jsonString = TestHelpers.getFileContents(fName); - - // Build DataPackage instance based on source file path. - new Package(jsonString, true); - - } - - @Test - void testCreateNews() throws Exception { - new Package( ); - }*/ -} diff --git a/src/test/java/io/frictionlessdata/datapackage/TestUtil.java b/src/test/java/io/frictionlessdata/datapackage/TestUtil.java index bb880b2..a996198 100644 --- a/src/test/java/io/frictionlessdata/datapackage/TestUtil.java +++ b/src/test/java/io/frictionlessdata/datapackage/TestUtil.java @@ -1,13 +1,22 @@ package io.frictionlessdata.datapackage; +import java.io.ByteArrayOutputStream; import java.io.File; -import java.net.URISyntaxException; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; import java.nio.file.Path; import java.nio.file.Paths; -class TestUtil { +public class TestUtil { - static Path getBasePath() { + public static File getTestDataDirectory()throws Exception { + URL u = TestUtil.class.getResource("/fixtures/multi_data_datapackage.json"); + Path path = Paths.get(u.toURI()); + return path.getParent().getParent().toFile(); + } + + public static Path getBasePath() { try { String pathName = "/fixtures/multi_data_datapackage.json"; Path sourceFileAbsPath = Paths.get(TestUtil.class.getResource(pathName).toURI()); @@ -16,4 +25,37 @@ static Path getBasePath() { throw new RuntimeException(ex); } } + + public static Path getResourcePath (String fileName) { + try { + String locFileName = fileName; + if (!fileName.startsWith("/")){ + locFileName = "/"+fileName; + } + // Create file-URL of source file: + URL sourceFileUrl = TestUtil.class.getResource(locFileName); + // Get path of URL + return Paths.get(sourceFileUrl.toURI()); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + public static byte[] getResourceContent (String fileName) throws IOException { + String locFileName = fileName; + if (fileName.startsWith("/")){ + locFileName = fileName.substring(1); + } + + try (InputStream inputStream = Package.class.getClassLoader().getResourceAsStream(locFileName); + ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (int b; (b = inputStream.read()) != -1; ) { + out.write(b); + } + return out.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } diff --git a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java index ad53e69..0331938 100644 --- a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java @@ -1,19 +1,18 @@ package io.frictionlessdata.datapackage; +import tools.jackson.databind.JsonNode; import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import java.io.IOException; +import io.frictionlessdata.tableschema.exception.ValidationException; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; -import org.everit.json.schema.ValidationException; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; -import static io.frictionlessdata.datapackage.TestUtil.getBasePath; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Test calls for JSON Validator class. @@ -22,46 +21,42 @@ public class ValidatorTest { private static URL url; - @Rule - public final ExpectedException exception = ExpectedException.none(); - - private Validator validator = null; - - @Before - public void setup() throws MalformedURLException { - validator = new Validator(); + @BeforeAll + public static void setup() throws MalformedURLException { url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/datapackages/multi-data/datapackage.json"); } @Test - public void testValidatingInvalidJsonObject() throws IOException, DataPackageException { - JSONObject datapackageJsonObject = new JSONObject("{\"invalid\" : \"json\"}"); - - exception.expect(ValidationException.class); - validator.validate(datapackageJsonObject); + @DisplayName("Test validating invalid JSON object as datapackage JSON throws an exception") + public void testValidatingInvalidJsonObject() throws DataPackageException { + JsonNode datapackageJsonObject = JsonUtil.getInstance().createNode("{\"invalid\" : \"json\"}"); + + assertThrows(ValidationException.class, () -> {Validator.validate(datapackageJsonObject);}); } @Test - public void testValidatingInvalidJsonString() throws IOException, DataPackageException{ + @DisplayName("Test validating invalid JSON string as datapackage JSON throws an exception") + public void testValidatingInvalidJsonString() throws DataPackageException{ String datapackageJsonString = "{\"invalid\" : \"json\"}"; - - exception.expect(ValidationException.class); - validator.validate(datapackageJsonString); + + assertThrows(ValidationException.class, () -> {Validator.validate(datapackageJsonString);}); } @Test + @DisplayName("Test setting an undefined type of profile on a datapackage throws an exception") public void testValidationWithInvalidProfileId() throws Exception { Package dp = new Package(url, true); String invalidProfileId = "INVALID_PROFILE_ID"; - dp.addProperty("profile", invalidProfileId); - - exception.expectMessage("Invalid profile id: " + invalidProfileId); - dp.validate(); + dp.setProfile(invalidProfileId); + + DataPackageException ex = assertThrows(DataPackageException.class, dp::validate); + Assertions.assertEquals("Invalid profile id: " + invalidProfileId, ex.getMessage()); } @Test + @DisplayName("Test setting a profile from an URL on a datapackage") public void testValidationWithValidProfileUrl() throws Exception { Package dp = new Package(url, true); dp.setProfile( "https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + @@ -70,19 +65,20 @@ public void testValidationWithValidProfileUrl() throws Exception { dp.validate(); // No exception thrown, test passes. - Assert.assertEquals("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/" + + Assertions.assertEquals("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/" + "master/src/main/resources/schemas/data-package.json", dp.getProfile()); } @Test + @DisplayName("Test setting a profile from an invalid URL on a datapackage throws an exception") public void testValidationWithInvalidProfileUrl() throws Exception { Package dp = new Package(url, true); String invalidProfileUrl = "https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/main/resources/schemas/INVALID.json"; - dp.addProperty("profile", invalidProfileUrl); - - exception.expectMessage("Invalid profile schema URL: " + invalidProfileUrl); - dp.validate(); + dp.setProfile(invalidProfileUrl); + + DataPackageException ex = assertThrows(DataPackageException.class, dp::validate); + Assertions.assertEquals("Invalid profile schema URL: " + invalidProfileUrl, ex.getMessage()); } } diff --git a/src/test/java/io/frictionlessdata/datapackage/beans/EmployeeBean.java b/src/test/java/io/frictionlessdata/datapackage/beans/EmployeeBean.java index 2ef7bed..cf8ceba 100644 --- a/src/test/java/io/frictionlessdata/datapackage/beans/EmployeeBean.java +++ b/src/test/java/io/frictionlessdata/datapackage/beans/EmployeeBean.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; -import org.json.JSONObject; import org.locationtech.jts.geom.Coordinate; import java.time.Duration; diff --git a/src/test/java/io/frictionlessdata/datapackage/beans/GrossDomesticProductBean.java b/src/test/java/io/frictionlessdata/datapackage/beans/GrossDomesticProductBean.java new file mode 100644 index 0000000..1c686e9 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/beans/GrossDomesticProductBean.java @@ -0,0 +1,39 @@ +package io.frictionlessdata.datapackage.beans; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; + +import java.math.BigDecimal; +import java.time.Year; + +@JsonPropertyOrder({ + "Country Name", "Country Code", "Year", "Value" +}) +public class GrossDomesticProductBean { + + //CSV header columns: + // Country Name,Country Code,Year,Value + + + @JsonProperty("Country Name") + String countryName; + + @JsonProperty("Country Code") + String countryCode; + + @JsonProperty("Year") + Year year; + + @JsonProperty("Value") + BigDecimal amount; + + @Override + public String toString() { + return "GrossDomesticProductBean{" + + "countryName='" + countryName + '\'' + + ", countryCode='" + countryCode + '\'' + + ", year=" + year + + ", amount=" + amount + + '}'; + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/beans/NumbersBean.java b/src/test/java/io/frictionlessdata/datapackage/beans/NumbersBean.java new file mode 100644 index 0000000..b909acf --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/beans/NumbersBean.java @@ -0,0 +1,217 @@ +package io.frictionlessdata.datapackage.beans; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.google.common.util.concurrent.AtomicDouble; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@JsonPropertyOrder({ + "id", "byteVal", "shortVal", "intVal", "longClassVal", "longVal", "floatVal", + "doubleVal", "floatClassVal", "doubleClassVal", "bigIntVal", "bigDecimalVal", + "atomicIntegerVal", "atomicLongVal", "atomicDoubleVal" +}) +public class NumbersBean { + + private Integer id; + + private byte byteVal; + + private short shortVal; + + private int intVal; + + private Long longClassVal; + + private long longVal; + + private float floatVal; + + private double doubleVal; + + private float floatClassVal; + + private double doubleClassVal; + + private BigInteger bigIntVal; + + private BigDecimal bigDecimalVal; + + private AtomicInteger atomicIntegerVal; + + private AtomicLong atomicLongVal; + + private AtomicDouble atomicDoubleVal; + + public Integer getId() { + return id; + } + + public void setId(Integer id) { + this.id = id; + } + + public byte getByteVal() { + return byteVal; + } + + public void setByteVal(byte byteVal) { + this.byteVal = byteVal; + } + + public short getShortVal() { + return shortVal; + } + + public void setShortVal(short shortVal) { + this.shortVal = shortVal; + } + + public int getIntVal() { + return intVal; + } + + public void setIntVal(int intVal) { + this.intVal = intVal; + } + + public Long getLongClassVal() { + return longClassVal; + } + + public void setLongClassVal(Long longClassVal) { + this.longClassVal = longClassVal; + } + + public long getLongVal() { + return longVal; + } + + public void setLongVal(long longVal) { + this.longVal = longVal; + } + + public float getFloatVal() { + return floatVal; + } + + public void setFloatVal(float floatVal) { + this.floatVal = floatVal; + } + + public double getDoubleVal() { + return doubleVal; + } + + public void setDoubleVal(double doubleVal) { + this.doubleVal = doubleVal; + } + + public BigInteger getBigIntVal() { + return bigIntVal; + } + + public void setBigIntVal(BigInteger bigIntVal) { + this.bigIntVal = bigIntVal; + } + + public BigDecimal getBigDecimalVal() { + return bigDecimalVal; + } + + public void setBigDecimalVal(BigDecimal bigDecimalVal) { + this.bigDecimalVal = bigDecimalVal; + } + + public AtomicInteger getAtomicIntegerVal() { + return atomicIntegerVal; + } + + public void setAtomicIntegerVal(AtomicInteger atomicIntegerVal) { + this.atomicIntegerVal = atomicIntegerVal; + } + + public AtomicLong getAtomicLongVal() { + return atomicLongVal; + } + + public void setAtomicLongVal(AtomicLong atomicLongVal) { + this.atomicLongVal = atomicLongVal; + } + + public AtomicDouble getAtomicDoubleVal() { + return atomicDoubleVal; + } + + public void setAtomicDoubleVal(AtomicDouble atomicDoubleVal) { + this.atomicDoubleVal = atomicDoubleVal; + } + + + public float getFloatClassVal() { + return floatClassVal; + } + + public void setFloatClassVal(float floatClassVal) { + this.floatClassVal = floatClassVal; + } + + public double getDoubleClassVal() { + return doubleClassVal; + } + + public void setDoubleClassVal(double doubleClassVal) { + this.doubleClassVal = doubleClassVal; + } + + @Override + public String toString() { + return "NumbersBean{" + + "id=" + id + + ", byteVal=" + byteVal + + ", shortVal=" + shortVal + + ", intVal=" + intVal + + ", longClassVal=" + longClassVal + + ", longVal=" + longVal + + ", floatVal=" + floatVal + + ", doubleVal=" + doubleVal + + ", floatClassVal=" + floatClassVal+ + ", doubleClassVal=" + doubleClassVal+ + ", bigIntVal=" + bigIntVal + + ", bigDecimalVal=" + bigDecimalVal + + ", atomicIntegerVal=" + atomicIntegerVal + + ", atomicLongVal=" + atomicLongVal + + ", atomicDoubleVal=" + atomicDoubleVal + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof NumbersBean)) return false; + NumbersBean that = (NumbersBean) o; + return byteVal == that.byteVal && + shortVal == that.shortVal && + intVal == that.intVal && + longVal == that.longVal && + Float.compare(that.floatVal, floatVal) == 0 && + Double.compare(that.doubleVal, doubleVal) == 0 && + Objects.equals(id, that.id) && + Objects.equals(longClassVal, that.longClassVal) && + Objects.equals(floatClassVal, that.floatClassVal)&& + Objects.equals(doubleClassVal, that.doubleClassVal)&& + Objects.equals(bigIntVal, that.bigIntVal) && + Objects.equals(bigDecimalVal, that.bigDecimalVal) && + Objects.equals(atomicIntegerVal.intValue(), that.atomicIntegerVal.intValue()) && + Objects.equals(atomicLongVal.longValue(), that.atomicLongVal.longValue()) && + Objects.equals(atomicDoubleVal.doubleValue(), that.atomicDoubleVal.doubleValue()); + } + + @Override + public int hashCode() { + return Objects.hash(id, byteVal, shortVal, intVal, longClassVal, longVal, floatVal, doubleVal, bigIntVal, bigDecimalVal, atomicIntegerVal, atomicLongVal, atomicDoubleVal); + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/fk/ForeignKeyTest.java b/src/test/java/io/frictionlessdata/datapackage/fk/ForeignKeyTest.java new file mode 100644 index 0000000..781a124 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/fk/ForeignKeyTest.java @@ -0,0 +1,54 @@ +package io.frictionlessdata.datapackage.fk; + +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.TestUtil; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.datapackage.resource.Resource; +import io.frictionlessdata.tableschema.exception.ForeignKeyException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class ForeignKeyTest { + + @Test + @DisplayName("Test that foreign keys are validated correctly, good case") + void testForeignKeysGoodCase() throws Exception{ + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/foreign_keys_valid.json"); + io.frictionlessdata.datapackage.Package pkg = new io.frictionlessdata.datapackage.Package(resourcePath, true); + Resource teams = pkg.getResource("teams"); + teams.checkRelations(pkg); + } + + @Test + @DisplayName("Test that foreign keys are validated correctly, bad case") + void testForeignKeysBadCase() throws Exception{ + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/foreign_keys_invalid.json"); + io.frictionlessdata.datapackage.Package pkg = new Package(resourcePath, true); + Resource teams = pkg.getResource("teams"); + + DataPackageValidationException ex = assertThrows(DataPackageValidationException.class, + () -> teams.checkRelations(pkg)); + Throwable cause = ex.getCause(); + Assertions.assertInstanceOf(ForeignKeyException.class, cause); + Assertions.assertEquals("Foreign key validation failed: [city] -> [name]: 'Munich' not found in resource 'cities'.", cause.getMessage()); + } + + @Test + @DisplayName("Test checkRelations on valid resources in different-valid-data-formats datapackage") + void testCheckRelationsOnAllResources() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/different-valid-data-formats/datapackage.json"); + Package dp = new Package(resourcePath, true); + + dp.getResource("teams_with_headers_csv_file").checkRelations(dp); + dp.getResource("teams_arrays_inline").checkRelations(dp); + dp.getResource("teams_objects_inline").checkRelations(dp); + dp.getResource("teams_arrays_file").checkRelations(dp); + dp.getResource("teams_objects_file").checkRelations(dp); + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java new file mode 100644 index 0000000..a77b48d --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/resource/JsonDataResourceTest.java @@ -0,0 +1,605 @@ +package io.frictionlessdata.datapackage.resource; + +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.PackageTest; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.math.BigInteger; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Year; +import java.util.*; + +import static io.frictionlessdata.datapackage.Profile.*; +import static io.frictionlessdata.datapackage.TestUtil.getTestDataDirectory; + +/** + * + * + */ +public class JsonDataResourceTest { + static ObjectNode resource1 = (ObjectNode) JsonUtil.getInstance().createNode("{\"name\": \"first-resource\", \"path\": " + + "[\"data/cities.csv\", \"data/cities2.csv\", \"data/cities3.csv\"]}"); + static ObjectNode resource2 = (ObjectNode) JsonUtil.getInstance().createNode("{\"name\": \"second-resource\", \"path\": " + + "[\"data/area.csv\", \"data/population.csv\"]}"); + + static ArrayNode testResources; + + static { + testResources = JsonUtil.getInstance().createArrayNode(); + testResources.add(resource1); + testResources.add(resource2); + } + + @Test + public void testIterateDataFromUrlPath() throws Exception{ + + String urlString = "https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + + "/master/src/test/resources/fixtures/data/population.csv"; + List dataSource = Arrays.asList(new URL(urlString)); + Resource resource = new URLbasedResource("population", dataSource); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get objectArrayIterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataFromFilePath() throws Exception{ + Resource resource = buildResource("/fixtures/data/population.csv"); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get objectArrayIterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataFromMultipartFilePath() throws Exception{ + List expectedData = new ArrayList(); + expectedData.add(new String[]{"libreville", "0.41,9.29"}); + expectedData.add(new String[]{"dakar", "14.71,-17.53"}); + expectedData.add(new String[]{"ouagadougou", "12.35,-1.67"}); + expectedData.add(new String[]{"barranquilla", "10.98,-74.88"}); + expectedData.add(new String[]{"rio de janeiro", "-22.91,-43.72"}); + expectedData.add(new String[]{"cuidad de guatemala", "14.62,-90.56"}); + expectedData.add(new String[]{"london", "51.50,-0.11"}); + expectedData.add(new String[]{"paris", "48.85,2.30"}); + expectedData.add(new String[]{"rome", "41.89,12.51"}); + + String[] paths = new String[]{ + "data/cities.csv", + "data/cities2.csv", + "data/cities3.csv"}; + List files = new ArrayList<>(); + for (String file : paths) { + files.add(new File (file)); + } + Resource resource = new FilebasedResource("coordinates", files, getBasePath()); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String coords = record[1]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], coords); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataFromMultipartURLPath() throws Exception{ + List expectedData = new ArrayList(); + expectedData.add(new String[]{"libreville", "0.41,9.29"}); + expectedData.add(new String[]{"dakar", "14.71,-17.53"}); + expectedData.add(new String[]{"ouagadougou", "12.35,-1.67"}); + expectedData.add(new String[]{"barranquilla", "10.98,-74.88"}); + expectedData.add(new String[]{"rio de janeiro", "-22.91,-43.72"}); + expectedData.add(new String[]{"cuidad de guatemala", "14.62,-90.56"}); + expectedData.add(new String[]{"london", "51.50,-0.11"}); + expectedData.add(new String[]{"paris", "48.85,2.30"}); + expectedData.add(new String[]{"rome", "41.89,12.51"}); + + String[] paths = new String[]{"https://raw.githubusercontent.com" + + "/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities.csv", + "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test" + + "/resources/fixtures/data/cities2.csv", + "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src" + + "/test/resources/fixtures/data/cities3.csv"}; + List urls = new ArrayList<>(); + for (String file : paths) { + urls.add(new URL (file)); + } + Resource resource = new URLbasedResource("coordinates", urls); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String coords = record[1]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], coords); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataWithCast() throws Exception{ + // Get string content version of the schema file. + String schemaJsonString = getFileContents("/fixtures/schema/population_schema.json"); + + Resource resource = buildResource("/fixtures/data/population.csv"); + + //set schema + resource.setSchema(Schema.fromJson(schemaJsonString, true)); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + Iterator iter = resource.objectArrayIterator(false, false); + + // Assert data. + while(iter.hasNext()){ + Object[] record = iter.next(); + + Assertions.assertEquals(String.class, record[0].getClass()); + Assertions.assertEquals(Year.class, record[1].getClass()); + Assertions.assertEquals(BigInteger.class, record[2].getClass()); + } + } + + + @Test + public void testIterateDataFromCsvFormat() throws Exception{ + String dataString = "city,year,population\nlondon,2017,8780000\nparis,2017,2240000\nrome,2017,2860000"; + Resource resource = new CSVDataResource("population", dataString); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get Iterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + + @Test + public void testBuildAndIterateDataFromCsvFormat() throws Exception{ + String dataString = getFileContents("/fixtures/resource/valid_csv_resource.json"); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get Iterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testBuildAndIterateDataFromTabseparatedCsvFormat() throws Exception{ + String dataString = getFileContents("/fixtures/resource/valid_csv_resource_tabseparated.json"); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get Iterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataFromJSONFormat() throws Exception{ + String jsonData = "[" + + "{" + + "\"city\": \"london\"," + + "\"year\": 2017," + + "\"population\": 8780000" + + "}," + + "{" + + "\"city\": \"paris\"," + + "\"year\": 2017," + + "\"population\": 2240000" + + "}," + + "{" + + "\"city\": \"rome\"," + + "\"year\": 2017," + + "\"population\": 2860000" + + "}" + + "]"; + + JSONDataResource resource = new JSONDataResource("population", jsonData); + + //set a schema to guarantee the ordering of properties + Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema.json"), true); + resource.setSchema(schema); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get Iterator. + Iterator iter = resource.stringArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ + String jsonData = "[" + + "{" + + "\"city\": \"london\"," + + "\"year\": 2017," + + "\"population\": 8780000" + + "}," + + "{" + + "\"city\": \"paris\"," + + "\"year\": 2017," + + "\"population\": 2240000" + + "}," + + "{" + + "\"city\": \"rome\"," + + "\"year\": 2017," + + "\"population\": 2860000" + + "}" + + "]"; + + JSONDataResource resource = new JSONDataResource("population", jsonData); + + //set a schema to guarantee the ordering of properties + Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema_alternate.json"), true); + resource.setSchema(schema); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Expected data. + List expectedData = this.getExpectedAlternatePopulationData(); + + // Get Iterator. + Iterator iter = resource.stringArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + // Test is invalid as the order of properties in a JSON object is not guaranteed (see spec) + @Test + public void testBuildAndIterateDataFromJSONFormat() throws Exception{ + String dataString = getFileContents("/fixtures/resource/valid_json_array_resource.json"); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); + + // Expected data. + List expectedData = this.getExpectedPopulationData(); + + // Get Iterator. + Iterator iter = resource.objectArrayIterator(); + int expectedDataIndex = 0; + + // Assert data. + while(iter.hasNext()){ + String[] record = iter.next(); + String city = record[0]; + String year = record[1]; + String population = record[2]; + + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); + + expectedDataIndex++; + } + } + + @Test + public void testRead() throws Exception{ + Resource resource = buildResource("/fixtures/data/population.csv"); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Assert + Assertions.assertEquals(3, resource.getData(false, false, false, false).size()); + } + + @Test + public void testReadFromZipFile() throws Exception{ + String sourceFileAbsPath = JsonDataResourceTest.class.getResource("/fixtures/zip/countries-and-currencies.zip").getPath(); + + Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); + Resource r = dp.getResource("currencies"); + + List data = r.getData(false, false, false, false); + Assertions.assertEquals(2, data.size()); + Object[] row1 = data.get(0); + Assertions.assertEquals("USD", row1[0]); + Assertions.assertEquals("US Dollar", row1[1]); + Assertions.assertEquals("$", row1[2]); + + Object[] row2 = data.get(1); + Assertions.assertEquals("GBP", row2[0]); + Assertions.assertEquals("Pound Sterling", row2[1]); + Assertions.assertEquals("£", row2[2]); + } + + @Test + public void testHeadings() throws Exception{ + Resource resource = buildResource("/fixtures/data/population.csv"); + + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + + // Assert + Assertions.assertEquals("city", resource.getHeaders()[0]); + Assertions.assertEquals("year", resource.getHeaders()[1]); + Assertions.assertEquals("population", resource.getHeaders()[2]); + } + + + @Test + @DisplayName("Paths in File-based resources must not be absolute") + /* + Test to verify https://specs.frictionlessdata.io/data-resource/#data-location : + POSIX paths (unix-style with / as separator) are supported for referencing local files, + with the security restraint that they MUST be relative siblings or children of the descriptor. + Absolute paths (/) and relative parent paths (…/) MUST NOT be used, + and implementations SHOULD NOT support these path types. + */ + public void readCreateInvalidResourceContainingAbsolutePaths() throws Exception{ + URI sourceFileAbsPathURI1 = PackageTest.class.getResource("/fixtures/data/cities.csv").toURI(); + URI sourceFileAbsPathURI2 = PackageTest.class.getResource("/fixtures/data/cities2.csv").toURI(); + File sourceFileAbsPathU1 = Paths.get(sourceFileAbsPathURI1).toAbsolutePath().toFile(); + File sourceFileAbsPathU2 = Paths.get(sourceFileAbsPathURI2).toAbsolutePath().toFile(); + + ArrayList files = new ArrayList<>(); + files.add(sourceFileAbsPathU1); + files.add(sourceFileAbsPathU2); + + Exception dpe = Assertions.assertThrows(DataPackageException.class, () -> { + new FilebasedResource("resource-one", files, getBasePath()); + }); + Assertions.assertEquals("Path entries for file-based Resources cannot be absolute", dpe.getMessage()); + } + + @Test + @DisplayName("Test reading Resource data rows as Map, ensuring we get values of " + + "the correct Schema Field type") + public void testReadMapped1() throws Exception{ + String[][] referenceData = new String[][]{ + {"city","year","population"}, + {"london","2017","8780000"}, + {"paris","2017","2240000"}, + {"rome","2017","2860000"}}; + Resource resource = buildResource("/fixtures/data/population.csv"); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/population_schema.json"), true); + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + List> mappedData = resource.getMappedData(false); + Assertions.assertEquals(3, mappedData.size()); + String[] headers = referenceData[0]; + //need to omit the table header in the referenceData + for (int i = 0; i < mappedData.size(); i++) { + String[] refRow = referenceData[i+1]; + Map testData = mappedData.get(i); + // ensure row size is correct + Assertions.assertEquals(refRow.length, testData.size()); + + // ensure we get the headers in the right sort order + ArrayList testDataColKeys = new ArrayList<>(testData.keySet()); + String[] testHeaders = testDataColKeys.toArray(new String[]{}); + Assertions.assertArrayEquals(headers, testHeaders); + + // validate values match and types are as expected + Assertions.assertEquals(refRow[0], testData.get(testDataColKeys.get(0))); //String value for city name + Assertions.assertEquals(Year.class, testData.get(testDataColKeys.get(1)).getClass()); + Assertions.assertEquals(refRow[1], ((Year)testData.get(testDataColKeys.get(1))).toString());//Year value for year + Assertions.assertEquals(BigInteger.class, testData.get(testDataColKeys.get(2)).getClass()); //String value for city name + Assertions.assertEquals(refRow[2], testData.get(testDataColKeys.get(2)).toString());//BigInteger value for population + } + } + @Test + @DisplayName("Test setting invalid 'profile' property, must throw") + public void testSetInvalidProfile() throws Exception { + Resource resource = buildResource("/fixtures/data/population.csv"); + + Assertions.assertThrows(DataPackageValidationException.class, + () -> resource.setProfile(PROFILE_DATA_PACKAGE_DEFAULT)); + Assertions.assertThrows(DataPackageValidationException.class, + () -> resource.setProfile(PROFILE_TABULAR_DATA_PACKAGE)); + Assertions.assertDoesNotThrow(() -> resource.setProfile(PROFILE_DATA_RESOURCE_DEFAULT)); + Assertions.assertDoesNotThrow(() -> resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE)); + } + + private static Resource buildResource(String relativeInPath) throws URISyntaxException { + URL sourceFileUrl = JsonDataResourceTest.class.getResource(relativeInPath); + Path path = Paths.get(sourceFileUrl.toURI()); + Path parent = path.getParent(); + Path relativePath = parent.relativize(path); + + List files = new ArrayList<>(); + files.add(relativePath.toFile()); + return new FilebasedResource("population", files, parent.toFile()); + } + + private static File getBasePath() throws URISyntaxException { + URL sourceFileUrl = JsonDataResourceTest.class.getResource("/fixtures/data"); + Path path = Paths.get(sourceFileUrl.toURI()); + return path.getParent().toFile(); + } + + private static String getFileContents(String fileName) { + try { + // Create file-URL of source file: + URL sourceFileUrl = JsonDataResourceTest.class.getResource(fileName); + // Get path of URL + Path path = Paths.get(sourceFileUrl.toURI()); + return new String(Files.readAllBytes(path)); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private List getExpectedPopulationData(){ + List expectedData = new ArrayList<>(); + //expectedData.add(new String[]{"city", "year", "population"}); + expectedData.add(new String[]{"london", "2017", "8780000"}); + expectedData.add(new String[]{"paris", "2017", "2240000"}); + expectedData.add(new String[]{"rome", "2017", "2860000"}); + + return expectedData; + } + + private List getExpectedAlternatePopulationData(){ + List expectedData = new ArrayList<>(); + expectedData.add(new String[]{"2017", "london", "8780000"}); + expectedData.add(new String[]{"2017", "paris", "2240000"}); + expectedData.add(new String[]{"2017", "rome", "2860000"}); + + return expectedData; + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/NonTabularResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/NonTabularResourceTest.java new file mode 100644 index 0000000..9661ee1 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/resource/NonTabularResourceTest.java @@ -0,0 +1,745 @@ +package io.frictionlessdata.datapackage.resource; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.*; +import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.exception.TypeInferringException; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.ArrayList; + +import static io.frictionlessdata.datapackage.Profile.PROFILE_DATA_PACKAGE_DEFAULT; +import static io.frictionlessdata.datapackage.Profile.PROFILE_DATA_RESOURCE_DEFAULT; + +public class NonTabularResourceTest { + + private static final String REFERENCE1 = "{\n" + + " \"name\" : \"employees\",\n" + + " \"profile\" : \"data-package\",\n" + + " \"resources\" : [ {\n" + + " \"name\" : \"employee-data\",\n" + + " \"profile\" : \"tabular-data-resource\",\n" + + " \"schema\" : \"schema.json\",\n" + + " \"path\" : \"data/employees.csv\"\n" + + " }, {\n" + + " \"name\" : \"non-tabular-resource\",\n" + + " \"profile\" : \"data-resource\",\n" + + " \"format\" : \"pdf\",\n" + + " \"path\" : \"data/sample.pdf\"\n" + + " } ]\n" + + "}"; + + private static final String REFERENCE2 = "{\n" + + " \"name\" : \"employees\",\n" + + " \"profile\" : \"data-package\",\n" + + " \"resources\" : [ {\n" + + " \"name\" : \"employee-data\",\n" + + " \"profile\" : \"tabular-data-resource\",\n" + + " \"schema\" : \"schema.json\",\n" + + " \"path\" : \"data/employees.csv\"\n" + + " }, {\n" + + " \"name\" : \"non-tabular-resource\",\n" + + " \"profile\" : \"data-resource\",\n" + + " \"format\" : \"json\",\n" + + " \"data\" : {\n" + + " \"name\" : \"John Doe\",\n" + + " \"age\" : 30\n" + + " }\n" + + " } ]\n" + + "}"; + + private static final String REFERENCE3 = "{\n" + + " \"name\" : \"employees\",\n" + + " \"profile\" : \"data-package\",\n" + + " \"resources\" : [ {\n" + + " \"name\" : \"employee-data\",\n" + + " \"profile\" : \"tabular-data-resource\",\n" + + " \"schema\" : \"schema.json\",\n" + + " \"path\" : \"data/employees.csv\"\n" + + " }, {\n" + + " \"name\" : \"non-tabular-resource\",\n" + + " \"profile\" : \"data-resource\",\n" + + " \"encoding\" : \"UTF-8\",\n" + + " \"format\" : \"pdf\",\n" + + " \"path\" : \"sample.pdf\"\n" + + " } ]\n" + + "}"; + + @Test + @DisplayName("Test adding a non-tabular resource, and saving package") + public void testNonTabularResource1() throws Exception { + Path tempDirPath = Files.createTempDirectory("datapackage-"); + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + io.frictionlessdata.datapackage.Package dp = new Package(resourcePath, true); + dp.setProfile(PROFILE_DATA_PACKAGE_DEFAULT); + + byte[] referenceData = TestUtil.getResourceContent("/fixtures/files/sample.pdf"); + Resource testResource = new myNonTabularResource(referenceData, Path.of("data/sample.pdf")); + testResource.setShouldSerializeToFile(true); + dp.addResource(testResource); + + // write the package with the new resource to the file system + File outFile = new File (tempDirPath.toFile(), "datapackage.json"); + dp.write(outFile, false); + + // read back the datapackage.json and compare to the expected output + String content = String.join("\n", Files.readAllLines(outFile.toPath())); + Assertions.assertEquals(REFERENCE1.replaceAll("[\n\r]+", "\n"), content.replaceAll("[\n\r]+", "\n")); + + // read the package back in and check number of resourcces + Package round2Package = new Package(outFile.toPath(), true); + Assertions.assertEquals(2, round2Package.getResources().size()); + + // compare the non-tabular resource with expected values + Resource round2Resource = round2Package.getResource("non-tabular-resource"); + Assertions.assertEquals("data-resource", round2Resource.getProfile()); + Assertions.assertArrayEquals(referenceData, (byte[]) round2Resource.getRawData()); + + // write the package out again + Path tempDirPathRound3 = Files.createTempDirectory("datapackage-"); + File outFileRound3 = new File (tempDirPathRound3.toFile(), "datapackage.json"); + round2Package.write(outFileRound3, false); + + // read back the datapackage.json and compare to the expected output + String contentRound3 = String.join("\n", Files.readAllLines(outFileRound3.toPath())); + Assertions.assertEquals(REFERENCE1.replaceAll("[\n\r]+", "\n"), contentRound3.replaceAll("[\n\r]+", "\n")); + + // read the package back in and check number of resourcces + Package round3Package = new Package(outFileRound3.toPath(), true); + Assertions.assertEquals(2, round3Package.getResources().size()); + + // compare the non-tabular resource with expected values + Resource round3Resource = round3Package.getResource("non-tabular-resource"); + Assertions.assertEquals("data-resource", round3Resource.getProfile()); + Object rawData = round3Resource.getRawData(); + Assertions.assertArrayEquals(referenceData, (byte[]) rawData); + } + + + @Test + @DisplayName("Test adding a non-tabular JSON resource, and saving package") + public void testNonTabularResource2() throws Exception { + Path tempDirPath = Files.createTempDirectory("datapackage-"); + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + io.frictionlessdata.datapackage.Package dp = new Package(resourcePath, true); + dp.setProfile(PROFILE_DATA_PACKAGE_DEFAULT); + + ObjectNode referenceData = (ObjectNode) JsonUtil.getInstance().createNode("{\"name\": \"John Doe\", \"age\": 30}"); + Resource testResource = new JSONObjectResource("non-tabular-resource", referenceData); + dp.addResource(testResource); + + File outFile = new File (tempDirPath.toFile(), "datapackage.json"); + dp.write(outFile, false); + + String content = String.join("\n", Files.readAllLines(outFile.toPath())); + Assertions.assertEquals(REFERENCE2.replaceAll("[\n\r]+", "\n"), content.replaceAll("[\n\r]+", "\n")); + + Package round2Package = new Package(outFile.toPath(), true); + Assertions.assertEquals(2, round2Package.getResources().size()); + Assertions.assertEquals("non-tabular-resource", round2Package.getResources().get(1).getName()); + Assertions.assertEquals("data-resource", round2Package.getResources().get(1).getProfile()); + Assertions.assertEquals("json", round2Package.getResources().get(1).getFormat()); + Assertions.assertEquals(referenceData, round2Package.getResources().get(1).getRawData()); + + // write the package out again + Path tempDirPathRound3 = Files.createTempDirectory("datapackage-"); + File outFileRound3 = new File (tempDirPathRound3.toFile(), "datapackage.json"); + round2Package.write(outFileRound3, false); + + // read back the datapackage.json and compare to the expected output + String contentRound3 = String.join("\n", Files.readAllLines(outFileRound3.toPath())); + Assertions.assertEquals(REFERENCE2.replaceAll("[\n\r]+", "\n"), contentRound3.replaceAll("[\n\r]+", "\n")); + + // read the package back in and check number of resourcces + Package round3Package = new Package(outFileRound3.toPath(), true); + Assertions.assertEquals(2, round3Package.getResources().size()); + + // compare the non-tabular resource with expected values + Resource round3Resource = round3Package.getResource("non-tabular-resource"); + Assertions.assertEquals("data-resource", round3Resource.getProfile()); + Object rawData = round3Resource.getRawData(); + Assertions.assertEquals(referenceData, rawData); + + } + + @Test + @DisplayName("Test adding a non-tabular file resource, and saving package") + public void testNonTabularResource3() throws Exception { + File tempDirPathData = Files.createTempDirectory("datapackage-").toFile(); + String fName = "/fixtures/datapackages/employees/datapackage.json"; + Path resourcePath = TestUtil.getResourcePath(fName); + io.frictionlessdata.datapackage.Package dp = new Package(resourcePath, true); + dp.setProfile(PROFILE_DATA_PACKAGE_DEFAULT); + + byte[] referenceData = TestUtil.getResourceContent("/fixtures/files/sample.pdf"); + String fileName = "sample.pdf"; + File f = new File(tempDirPathData, fileName); + try (OutputStream os = Files.newOutputStream(f.toPath())) { + os.write(referenceData); + } catch (IOException e) { + throw new RuntimeException("Error writing DOE setup to file: " + e.getMessage(), e); + } + FilebasedResource testResource = new FilebasedResource("non-tabular-resource", List.of(new File(fileName)), tempDirPathData); + testResource.setProfile(Profile.PROFILE_DATA_RESOURCE_DEFAULT); + testResource.setSerializationFormat(null); + dp.addResource(testResource); + + File tempDirPath = Files.createTempDirectory("datapackage-").toFile(); + File outFile = new File (tempDirPath, "datapackage.json"); + dp.write(outFile, false); + + String content = String.join("\n", Files.readAllLines(outFile.toPath())); + Assertions.assertEquals(REFERENCE3.replaceAll("[\n\r]+", "\n"), content.replaceAll("[\n\r]+", "\n")); + + Package round2Package = new Package(outFile.toPath(), true); + Assertions.assertEquals(2, round2Package.getResources().size()); + Assertions.assertEquals("non-tabular-resource", round2Package.getResources().get(1).getName()); + Assertions.assertEquals("data-resource", round2Package.getResources().get(1).getProfile()); + Assertions.assertEquals("pdf", round2Package.getResources().get(1).getFormat()); + Assertions.assertEquals(new String((byte[])referenceData), new String((byte[])round2Package.getResources().get(1).getRawData())); + + // write the package out again + Path tempDirPathRound3 = Files.createTempDirectory("datapackage-"); + File outFileRound3 = new File (tempDirPathRound3.toFile(), "datapackage.json"); + round2Package.write(outFileRound3, false); + + // read back the datapackage.json and compare to the expected output + String contentRound3 = String.join("\n", Files.readAllLines(outFileRound3.toPath())); + Assertions.assertEquals(REFERENCE3.replaceAll("[\n\r]+", "\n"), contentRound3.replaceAll("[\n\r]+", "\n")); + + // read the package back in and check number of resourcces + Package round3Package = new Package(outFileRound3.toPath(), true); + Assertions.assertEquals(2, round3Package.getResources().size()); + + // compare the non-tabular resource with expected values + Resource round3Resource = round3Package.getResource("non-tabular-resource"); + Assertions.assertEquals("data-resource", round3Resource.getProfile()); + Object rawData = round3Resource.getRawData(); + Assertions.assertEquals(new String((byte[])referenceData), new String((byte[])rawData)); + + } + + + @Test + @DisplayName("Test creating ZIP-compressed datapackage with PDF as FilebasedResource") + public void testZipCompressedDatapackageWithPdfResource() throws Exception { + // Create temporary directories + Path tempDirPath = Files.createTempDirectory("datapackage-source-"); + Path tempOutputPath = Files.createTempDirectory("datapackage-output-"); + + // Copy PDF to temporary directory + byte[] pdfContent = TestUtil.getResourceContent("/fixtures/files/sample.pdf"); + File pdfFile = new File(tempDirPath.toFile(), "sample.pdf"); + Files.write(pdfFile.toPath(), pdfContent); + + // Create FilebasedResource directly for the PDF + FilebasedResource pdfResource = new FilebasedResource( + "pdf-document", + List.of(new File("sample.pdf")), + tempDirPath.toFile() + ); + + // Create a package with the resource + List resources = new ArrayList<>(); + resources.add(pdfResource); + io.frictionlessdata.datapackage.Package pkg = new io.frictionlessdata.datapackage.Package(resources); + pkg.setName("pdf-package"); + pkg.setProfile(Profile.PROFILE_DATA_PACKAGE_DEFAULT); + + // Set default resource profile and format + pdfResource.setProfile(Profile.PROFILE_DATA_RESOURCE_DEFAULT); + pdfResource.setFormat("pdf"); + pdfResource.setTitle("Sample PDF Document"); + pdfResource.setDescription("A test PDF file"); + pdfResource.setMediaType("application/pdf"); + pdfResource.setEncoding("UTF-8"); + + // Write as ZIP-compressed package + File zipFile = new File(tempOutputPath.toFile(), "datapackage.zip"); + pkg.write(zipFile, true); // true for compression + + // Verify ZIP file was created + Assertions.assertTrue(zipFile.exists()); + Assertions.assertTrue(zipFile.length() > 0); + + // Read the package back from ZIP + io.frictionlessdata.datapackage.Package readPackage = new io.frictionlessdata.datapackage.Package(zipFile.toPath(), true); + + // Verify package properties + Assertions.assertEquals("pdf-package", readPackage.getName()); + Assertions.assertEquals(1, readPackage.getResources().size()); + + // Verify PDF resource + Resource readResource = readPackage.getResource("pdf-document"); + Assertions.assertNotNull(readResource); + Assertions.assertEquals(Profile.PROFILE_DATA_RESOURCE_DEFAULT, readResource.getProfile()); + Assertions.assertEquals("pdf", readResource.getFormat()); + Assertions.assertEquals("Sample PDF Document", readResource.getTitle()); + Assertions.assertEquals("application/pdf", readResource.getMediaType()); + } + + @Test + @DisplayName("Test creating uncompressed datapackage with PDF as FilebasedResource") + public void testUnCompressedDatapackageWithPdfResource() throws Exception { + // Create temporary directories + Path tempDirPath = Files.createTempDirectory("datapackage-source-"); + Path tempOutputPath = Files.createTempDirectory("datapackage-output-"); + + // Copy PDF to temporary directory + byte[] pdfContent = TestUtil.getResourceContent("/fixtures/files/sample.pdf"); + File pdfFile = new File(tempDirPath.toFile(), "sample.pdf"); + Files.write(pdfFile.toPath(), pdfContent); + + // Create FilebasedResource directly for the PDF + FilebasedResource pdfResource = new FilebasedResource( + "pdf-document", + List.of(new File("sample.pdf")), + tempDirPath.toFile() + ); + + // Create a package with the resource + List resources = new ArrayList<>(); + resources.add(pdfResource); + io.frictionlessdata.datapackage.Package pkg = new io.frictionlessdata.datapackage.Package(resources); + pkg.setName("pdf-package"); + pkg.setProfile(Profile.PROFILE_DATA_PACKAGE_DEFAULT); + + // Set default resource profile and format + pdfResource.setProfile(Profile.PROFILE_DATA_RESOURCE_DEFAULT); + pdfResource.setFormat("pdf"); + pdfResource.setTitle("Sample PDF Document"); + pdfResource.setDescription("A test PDF file"); + pdfResource.setMediaType("application/pdf"); + pdfResource.setEncoding("UTF-8"); + + // Write as ZIP-compressed package + File packageFile = new File(tempOutputPath.toFile(), "datapackage"); + pkg.write(packageFile, false); // false for compression + + Assertions.assertTrue(packageFile.exists());; + + // Read the package back from ZIP + io.frictionlessdata.datapackage.Package readPackage = new io.frictionlessdata.datapackage.Package(packageFile.toPath(), true); + + // Verify package properties + Assertions.assertEquals("pdf-package", readPackage.getName()); + Assertions.assertEquals(1, readPackage.getResources().size()); + + // Verify PDF resource + Resource readResource = readPackage.getResource("pdf-document"); + Assertions.assertNotNull(readResource); + Assertions.assertEquals(Profile.PROFILE_DATA_RESOURCE_DEFAULT, readResource.getProfile()); + Assertions.assertEquals("pdf", readResource.getFormat()); + Assertions.assertEquals("Sample PDF Document", readResource.getTitle()); + Assertions.assertEquals("application/pdf", readResource.getMediaType()); + } + + @Test + @DisplayName("Test creating ZIP-compressed datapackage with JSON ObjectNode as FilebasedResource") + public void testCreateAndReadZippedPackageWithJsonObject() throws Exception { + // Create temporary directories for source and output + Path tempDirPath = Files.createTempDirectory("datapackage-source-"); + Path locPath = tempDirPath.resolve("data"); + Files.createDirectories(locPath); + //Path tempOutputPath = Files.createTempDirectory("datapackage-output-"); + Path tempOutputPath = tempDirPath; + + // Create an ObjectNode and write it to a temporary file + ObjectMapper mapper = new ObjectMapper(); + ObjectNode jsonData = mapper.createObjectNode(); + jsonData.put("key", "value"); + jsonData.put("number", 123); + byte[] jsonBytes = mapper.writeValueAsBytes(jsonData); + + File jsonFile = new File(locPath.toFile(), "data.json"); + Files.write(jsonFile.toPath(), jsonBytes); + + // Create a FilebasedResource for the JSON file + FilebasedResource jsonResource = new FilebasedResource( + "json-data", + List.of(new File("data/data.json")), + tempDirPath.toFile() + ); + + // Set resource properties + jsonResource.setProfile(Profile.PROFILE_DATA_RESOURCE_DEFAULT); + jsonResource.setFormat("json"); + jsonResource.setMediaType("application/json"); + jsonResource.setEncoding("UTF-8"); + + // Create a package with the resource + List resources = new ArrayList<>(); + resources.add(jsonResource); + Package pkg = new Package(resources); + pkg.setName("json-package"); + + // Write the package to a compressed ZIP file + File zipFile = new File(tempOutputPath.toFile(), "datapackage.zip"); + pkg.write(zipFile, true); + + // Verify the ZIP file was created + Assertions.assertTrue(zipFile.exists()); + Assertions.assertTrue(zipFile.length() > 0); + + // Read the package back from the ZIP file + Package readPackage = new Package(zipFile.toPath(), true); + + // Verify package properties + Assertions.assertEquals("json-package", readPackage.getName()); + Assertions.assertEquals(1, readPackage.getResources().size()); + + // Verify the JSON resource + Resource readResource = readPackage.getResource("json-data"); + Assertions.assertNotNull(readResource); + Assertions.assertEquals("application/json", readResource.getMediaType()); + + // Verify the content of the resource + byte[] readData = (byte[]) readResource.getRawData(); + Assertions.assertArrayEquals(jsonBytes, readData); + Assertions.assertEquals("json", readResource.getFormat()); + } + + + /** + * A non-tabular resource + */ + + public class myNonTabularResource extends JSONBase implements Resource { + + private Object data; + + private final Path fileName; + + public myNonTabularResource(Object data, Path relativeOutPath) { + this.data = data; + super.setName("non-tabular-resource"); + this.fileName = relativeOutPath; + } + + @Override + @JsonIgnore + public List
getTables() { + return null; + } + + @Override + @JsonIgnore + public Object getRawData() { + return data; + } + + @Override + public List getData(boolean b) throws Exception { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public List> getMappedData(boolean b) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public List getData(boolean b, boolean b1, boolean b2, boolean b3) throws Exception { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @JsonProperty(JSON_KEY_PATH) + public String getPath() { + if (File.separator.equals("\\")) + return fileName.toString().replaceAll("\\\\", "/"); + return fileName.toString(); + } + + @Override + @JsonIgnore + public String getDataAsJson() { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public String getDataAsCsv() { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public String getDataAsCsv(Dialect dialect, Schema schema) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public List getData(Class aClass) throws Exception { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public void writeData(Path path) throws Exception { + Path p = path.resolve(fileName); + + try { + Files.write(p, (byte[])getRawData()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void writeData(Writer writer) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public void writeSchema(Path path) { + } + + @Override + public void writeDialect(Path parentFilePath) throws IOException { + } + + @Override + public Iterator objectArrayIterator() { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public Iterator objectArrayIterator(boolean b, boolean b1) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public Iterator> mappingIterator(boolean b) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public Iterator beanIterator(Class aClass, boolean b) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public Iterator stringArrayIterator() { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public Iterator stringArrayIterator(boolean b) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public String[] getHeaders() { + return null; + } + + @Override + public String getPathForWritingSchema() { + return null; + } + + @Override + @JsonIgnore + public String getPathForWritingDialect() { + return null; + } + + @Override + @JsonIgnore + public Set getDatafileNamesForWriting() { + return null; + } + + @Override + public String getName() { + return super.getName(); + } + + @Override + public void setName(String s) { + super.setName(s); + } + + @Override + public String getProfile() { + return PROFILE_DATA_RESOURCE_DEFAULT; + } + + @Override + public void setProfile(String s) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public String getTitle() { + return super.getTitle(); + } + + @Override + public void setTitle(String s) { + super.setTitle(s); + } + + @Override + public String getDescription() { + return super.getDescription(); + } + + @Override + public void setDescription(String s) { + super.setDescription(s); + } + + @Override + public String getMediaType() { + return super.getMediaType(); + } + + @Override + public void setMediaType(String s) { + super.setMediaType(s); + } + + @Override + public String getEncoding() { + return super.getEncoding(); + } + + @Override + public void setEncoding(String s) { + super.setEncoding(s); + } + + @Override + @JsonIgnore + public Integer getBytes() { + return super.getBytes(); + } + + @Override + public void setBytes(Integer integer) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public String getHash() { + return super.getHash(); + } + + @Override + public void setHash(String s) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public Dialect getDialect() { + return null; + } + + @Override + public void setDialect(Dialect dialect) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public String getFormat() { + return "pdf"; + } + + @Override + public void setFormat(String s) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public String getDialectReference() { + return null; + } + + @Override + @JsonIgnore + public Schema getSchema() { + return null; + } + + @Override + public void setSchema(Schema schema) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + public Schema inferSchema() throws TypeInferringException { + return null; + } + + @Override + public List getSources() { + return super.getSources(); + } + + @Override + public void setSources(List sources) { + super.setSources(sources); + } + + @Override + public List getLicenses() { + return super.getLicenses(); + } + + @Override + public void setLicenses(List licenses) { + super.setLicenses(licenses); + } + + @Override + public boolean shouldSerializeToFile() { + return true; + } + + @Override + public void setShouldSerializeToFile(boolean b) { + } + + @Override + public void setSerializationFormat(String s) { + throw new UnsupportedOperationException("Not supported on non-tabular Resources"); + } + + @Override + @JsonIgnore + public String getSerializationFormat() { + return null; + } + + @Override + public void checkRelations(Package aPackage) throws Exception { + } + + @Override + public void validate(Package aPackage) { + } + + } + +} diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceBuilderTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceBuilderTest.java new file mode 100644 index 0000000..5f48962 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceBuilderTest.java @@ -0,0 +1,164 @@ +package io.frictionlessdata.datapackage.resource; + +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.tableschema.schema.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + + + +public class ResourceBuilderTest { + @Test + @DisplayName("Test ResourceBuilder with two file paths") + public void testResourceBuilderWithTwoFiles() throws Exception { + // Prepare test data + String[] paths = new String[]{ + "datapackages/multi-data/data/cities.csv", + "datapackages/multi-data/data/cities2.csv"}; + File basePath = getBasePath(); + + // Build resource using ResourceBuilder + Resource resource = ResourceBuilder.create("city-coordinates") + .withFiles(basePath, paths) + .profile(Profile.PROFILE_TABULAR_DATA_RESOURCE) + .title("City Coordinates") + .description("Geographic coordinates of various cities") + .encoding("UTF-8") + .format(Resource.FORMAT_CSV) + .build(); + + // Verify resource properties + Assertions.assertEquals("city-coordinates", resource.getName()); + Assertions.assertEquals(Profile.PROFILE_TABULAR_DATA_RESOURCE, resource.getProfile()); + Assertions.assertEquals("City Coordinates", resource.getTitle()); + Assertions.assertEquals("Geographic coordinates of various cities", resource.getDescription()); + Assertions.assertEquals("UTF-8", resource.getEncoding()); + Assertions.assertEquals(Resource.FORMAT_CSV, resource.getFormat()); + + // Verify the resource is file-based and has correct paths + Assertions.assertInstanceOf(FilebasedResource.class, resource); + FilebasedResource fileResource = (FilebasedResource) resource; + + // Test that we can iterate through data from both files + Iterator iter = resource.objectArrayIterator(); + int recordCount = 0; + while (iter.hasNext()) { + Object[] record = iter.next(); + Assertions.assertEquals(2, record.length); // city and location columns + recordCount++; + } + + // Should have 6 records total (3 from each file) + Assertions.assertEquals(6, recordCount); + } + + @Test + @DisplayName("Test ResourceBuilder with multiple files and schema inference") + public void testResourceBuilderWithMultipleFilesAndSchemaInference() throws Exception { + // Prepare test files + String[] paths = new String[]{"data/cities.csv", "data/cities2.csv"}; + File basePath = getBasePath(); + List files = new ArrayList<>(); + for (String path : paths) { + files.add(new File(path)); + } + + // Build resource using ResourceBuilder with schema inference + Resource resource = ResourceBuilder.create("multi-cities") + .withFiles(getBasePath(), files) + .inferSchema() + .build(); + + // Verify resource was created correctly + Assertions.assertNotNull(resource); + Assertions.assertEquals("multi-cities", resource.getName()); + Assertions.assertInstanceOf(FilebasedResource.class, resource); + + // Verify schema was inferred + Schema inferredSchema = resource.getSchema(); + Assertions.assertNotNull(inferredSchema, "Schema should be inferred"); + Assertions.assertEquals(2, inferredSchema.getFields().size(), "Should have 2 fields"); + + // Verify field names from inferred schema + Assertions.assertEquals("city", inferredSchema.getFields().get(0).getName()); + Assertions.assertEquals("location", inferredSchema.getFields().get(1).getName()); + + // Verify data can be read correctly with the inferred schema + List data = resource.getData(false, false, false, false); + Assertions.assertEquals(6, data.size(), "Should have 6 rows total from both files"); + + // Verify first row from first file + Object[] firstRow = (Object[])data.get(0); + Assertions.assertEquals("libreville", firstRow[0]); + Assertions.assertEquals("0.41,9.29", firstRow[1]); + + // Verify first row from second file (4th row overall) + Object[] fourthRow =(Object[]) data.get(3); + Assertions.assertEquals("barranquilla", fourthRow[0]); + Assertions.assertEquals("10.98,-74.88", fourthRow[1]); + } + + @Test + @DisplayName("Create non-tabular resource with PDF file using ResourceBuilder") + public void testCreateNonTabularResourceWithPdfFile() throws Exception { + String fileName = "sample.pdf"; + // Get the PDF file path + File pdf = new File(fileName); + File basePath = new File(getBasePath(), "files"); + + // Build a non-tabular resource with PDF + Resource resource = ResourceBuilder.create("sample-pdf") + .withFile(basePath, pdf) + .format(null) + .profile(Profile.PROFILE_DATA_RESOURCE_DEFAULT) + .title("Sample PDF Document") + .description("A sample PDF file for testing non-tabular resources") + .mediaType("application/pdf") + .build(); + + // Verify the resource properties + Assertions.assertNotNull(resource); + Assertions.assertEquals("sample-pdf", resource.getName()); + Assertions.assertEquals("pdf", resource.getFormat()); + Assertions.assertEquals(Profile.PROFILE_DATA_RESOURCE_DEFAULT, resource.getProfile()); + Assertions.assertEquals("Sample PDF Document", resource.getTitle()); + Assertions.assertEquals("A sample PDF file for testing non-tabular resources", resource.getDescription()); + Assertions.assertEquals("application/pdf", resource.getMediaType()); + + // Verify it's a file-based resource + Assertions.assertInstanceOf(FilebasedResource.class, resource); + + File tempDirPath = Files.createTempDirectory("datapackage-").toFile(); + File outFile = new File(tempDirPath, fileName); + + // Write the resource to a file + resource.writeData(tempDirPath.toPath()); + Assertions.assertTrue(outFile.exists()); + Assertions.assertTrue(outFile.length() > 0, "Output file should not be empty"); + // read the file back to verify content + String content = new String(Files.readAllBytes(outFile.toPath()), StandardCharsets.UTF_8); + // compare the content with the original PDF file + String originalContent = new String(Files.readAllBytes(new File(basePath, fileName).toPath()), StandardCharsets.UTF_8); + Assertions.assertEquals(originalContent, content, "Content of the written PDF should match the original"); + + } + + private static File getBasePath() throws URISyntaxException { + URL sourceFileUrl = ResourceBuilderTest.class.getResource("/fixtures/data"); + Path path = Paths.get(sourceFileUrl.toURI()); + return path.getParent().toFile(); + } +} diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java index c95cb26..fc1c8a7 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java @@ -1,5 +1,19 @@ package io.frictionlessdata.datapackage.resource; +import tools.jackson.databind.node.ArrayNode; +import tools.jackson.databind.node.ObjectNode; +import io.frictionlessdata.datapackage.Dialect; +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.PackageTest; +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.datapackage.exceptions.DataPackageException; +import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.util.JsonUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + import java.io.File; import java.math.BigInteger; import java.net.URI; @@ -9,44 +23,29 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Year; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; +import java.util.*; -import io.frictionlessdata.datapackage.Package; -import io.frictionlessdata.datapackage.PackageTest; -import io.frictionlessdata.datapackage.Profile; -import io.frictionlessdata.datapackage.exceptions.DataPackageException; -import io.frictionlessdata.tableschema.schema.Schema; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import static io.frictionlessdata.datapackage.Profile.*; +import static io.frictionlessdata.datapackage.TestUtil.getTestDataDirectory; /** * * */ public class ResourceTest { - static JSONObject resource1 = new JSONObject("{\"name\": \"first-resource\", \"path\": " + + static ObjectNode resource1 = (ObjectNode) JsonUtil.getInstance().createNode("{\"name\": \"first-resource\", \"path\": " + "[\"data/cities.csv\", \"data/cities2.csv\", \"data/cities3.csv\"]}"); - static JSONObject resource2 = new JSONObject("{\"name\": \"second-resource\", \"path\": " + + static ObjectNode resource2 = (ObjectNode) JsonUtil.getInstance().createNode("{\"name\": \"second-resource\", \"path\": " + "[\"data/area.csv\", \"data/population.csv\"]}"); - static JSONArray testResources; + static ArrayNode testResources; static { - testResources = new JSONArray(); - testResources.put(resource1); - testResources.put(resource2); + testResources = JsonUtil.getInstance().createArrayNode(); + testResources.add(resource1); + testResources.add(resource2); } - @Rule - public final ExpectedException exception = ExpectedException.none(); - @Test public void testIterateDataFromUrlPath() throws Exception{ @@ -72,9 +71,9 @@ public void testIterateDataFromUrlPath() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -101,9 +100,9 @@ public void testIterateDataFromFilePath() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -144,8 +143,8 @@ public void testIterateDataFromMultipartFilePath() throws Exception{ String city = record[0]; String coords = record[1]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], coords); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], coords); expectedDataIndex++; } @@ -175,9 +174,11 @@ public void testIterateDataFromMultipartURLPath() throws Exception{ urls.add(new URL (file)); } Resource resource = new URLbasedResource("coordinates", urls); - + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); // Set the profile to tabular data resource. resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + Iterator iter = resource.objectArrayIterator(); int expectedDataIndex = 0; @@ -188,8 +189,8 @@ public void testIterateDataFromMultipartURLPath() throws Exception{ String city = record[0]; String coords = record[1]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], coords); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], coords); expectedDataIndex++; } @@ -208,17 +209,18 @@ public void testIterateDataWithCast() throws Exception{ // Set the profile to tabular data resource. resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); - Iterator iter = resource.objectArrayIterator(false, false, true, false); + Iterator iter = resource.objectArrayIterator(false, false); // Assert data. while(iter.hasNext()){ Object[] record = iter.next(); - Assert.assertEquals(String.class, record[0].getClass()); - Assert.assertEquals(Year.class, record[1].getClass()); - Assert.assertEquals(BigInteger.class, record[2].getClass()); + Assertions.assertEquals(String.class, record[0].getClass()); + Assertions.assertEquals(Year.class, record[1].getClass()); + Assertions.assertEquals(BigInteger.class, record[2].getClass()); } } + @Test public void testIterateDataFromCsvFormat() throws Exception{ @@ -242,9 +244,9 @@ public void testIterateDataFromCsvFormat() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -254,7 +256,7 @@ public void testIterateDataFromCsvFormat() throws Exception{ @Test public void testBuildAndIterateDataFromCsvFormat() throws Exception{ String dataString = getFileContents("/fixtures/resource/valid_csv_resource.json"); - Resource resource = Resource.build(new JSONObject(dataString), getBasePath(), false); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); // Expected data. List expectedData = this.getExpectedPopulationData(); @@ -263,6 +265,9 @@ public void testBuildAndIterateDataFromCsvFormat() throws Exception{ Iterator iter = resource.objectArrayIterator(); int expectedDataIndex = 0; + // check that data was read + Assertions.assertTrue(iter.hasNext()); + // Assert data. while(iter.hasNext()){ String[] record = iter.next(); @@ -270,9 +275,9 @@ public void testBuildAndIterateDataFromCsvFormat() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -281,7 +286,7 @@ public void testBuildAndIterateDataFromCsvFormat() throws Exception{ @Test public void testBuildAndIterateDataFromTabseparatedCsvFormat() throws Exception{ String dataString = getFileContents("/fixtures/resource/valid_csv_resource_tabseparated.json"); - Resource resource = Resource.build(new JSONObject(dataString), getBasePath(), false); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); // Expected data. List expectedData = this.getExpectedPopulationData(); @@ -297,9 +302,9 @@ public void testBuildAndIterateDataFromTabseparatedCsvFormat() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -325,7 +330,7 @@ public void testIterateDataFromJSONFormat() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema.json"), true); @@ -348,9 +353,9 @@ public void testIterateDataFromJSONFormat() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -376,7 +381,7 @@ public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ "}" + "]"; - JSONDataResource resource = new JSONDataResource<>("population", jsonData); + JSONDataResource resource = new JSONDataResource("population", jsonData); //set a schema to guarantee the ordering of properties Schema schema = Schema.fromJson(new File(getBasePath(), "/schema/population_schema_alternate.json"), true); @@ -399,9 +404,9 @@ public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } @@ -411,9 +416,9 @@ public void testIterateDataFromJSONFormatAlternateSchema() throws Exception{ @Test public void testBuildAndIterateDataFromJSONFormat() throws Exception{ String dataString = getFileContents("/fixtures/resource/valid_json_array_resource.json"); - Resource resource = Resource.build(new JSONObject(dataString), getBasePath(), false); + Resource resource = Resource.fromJSON((ObjectNode) JsonUtil.getInstance().createNode(dataString), getBasePath(), false); - // Expected data. + // Expected data. List expectedData = this.getExpectedPopulationData(); // Get Iterator. @@ -427,43 +432,14 @@ public void testBuildAndIterateDataFromJSONFormat() throws Exception{ String year = record[1]; String population = record[2]; - Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city); - Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year); - Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[0], city); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[1], year); + Assertions.assertEquals(expectedData.get(expectedDataIndex)[2], population); expectedDataIndex++; } } - - /* - FIXME: since strongly typed, those don't work anymore - @Test - public void testCreatingJSONResourceWithInvalidFormatNullValue() throws Exception { - URL url = new URL("https://raw.githubusercontent.com/frictionlessdata/" + - "datapackage-java/master/src/test/resources/fixtures/multi_data_datapackage.json"); - Package dp = new Package(url, true); - - // format property is null but data is not null. - Resource resource = new JSONDataResource("resource-name", testResources, (String)null); - - exception.expectMessage("Invalid Resource. The data and format properties cannot be null."); - dp.addResource(resource); - } - - @Test - public void testCreatingResourceWithInvalidFormatDataValue() throws Exception { - URL url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/multi_data_datapackage.json"); - Package dp = new Package(url, true); - - // data property is null but format is not null. - Resource resource = new JSONDataResource("resource-name", (String)null, "csv"); - - exception.expectMessage("Invalid Resource. The path property or the data and format properties cannot be null."); - dp.addResource(resource); - } - - */ @Test public void testRead() throws Exception{ Resource resource = buildResource("/fixtures/data/population.csv"); @@ -472,10 +448,9 @@ public void testRead() throws Exception{ resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); // Assert - Assert.assertEquals(3, resource.read(false).size()); + Assertions.assertEquals(3, resource.getData(false, false, false, false).size()); } - @Test public void testReadFromZipFile() throws Exception{ String sourceFileAbsPath = ResourceTest.class.getResource("/fixtures/zip/countries-and-currencies.zip").getPath(); @@ -483,7 +458,17 @@ public void testReadFromZipFile() throws Exception{ Package dp = new Package(new File(sourceFileAbsPath).toPath(), true); Resource r = dp.getResource("currencies"); - List data = r.read(false); + List data = r.getData(false, false, false, false); + Assertions.assertEquals(2, data.size()); + Object[] row1 = data.get(0); + Assertions.assertEquals("USD", row1[0]); + Assertions.assertEquals("US Dollar", row1[1]); + Assertions.assertEquals("$", row1[2]); + + Object[] row2 = data.get(1); + Assertions.assertEquals("GBP", row2[0]); + Assertions.assertEquals("Pound Sterling", row2[1]); + Assertions.assertEquals("£", row2[2]); } @Test @@ -494,30 +479,181 @@ public void testHeadings() throws Exception{ resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); // Assert - Assert.assertEquals("city", resource.getHeaders()[0]); - Assert.assertEquals("year", resource.getHeaders()[1]); - Assert.assertEquals("population", resource.getHeaders()[2]); + Assertions.assertEquals("city", resource.getHeaders()[0]); + Assertions.assertEquals("year", resource.getHeaders()[1]); + Assertions.assertEquals("population", resource.getHeaders()[2]); } @Test + @DisplayName("Paths in File-based resources must not be absolute") + /* + Test to verify https://specs.frictionlessdata.io/data-resource/#data-location : + POSIX paths (unix-style with / as separator) are supported for referencing local files, + with the security restraint that they MUST be relative siblings or children of the descriptor. + Absolute paths (/) and relative parent paths (…/) MUST NOT be used, + and implementations SHOULD NOT support these path types. + */ public void readCreateInvalidResourceContainingAbsolutePaths() throws Exception{ - Path tempDirPath = Files.createTempDirectory("datapackage-"); URI sourceFileAbsPathURI1 = PackageTest.class.getResource("/fixtures/data/cities.csv").toURI(); URI sourceFileAbsPathURI2 = PackageTest.class.getResource("/fixtures/data/cities2.csv").toURI(); File sourceFileAbsPathU1 = Paths.get(sourceFileAbsPathURI1).toAbsolutePath().toFile(); File sourceFileAbsPathU2 = Paths.get(sourceFileAbsPathURI2).toAbsolutePath().toFile(); + ArrayList files = new ArrayList<>(); files.add(sourceFileAbsPathU1); files.add(sourceFileAbsPathU2); - exception.expect(DataPackageException.class); - FilebasedResource r = new FilebasedResource("resource-one", files, getBasePath()); - Package pkg = new Package("test", tempDirPath.resolve("datapackage.json"), true); - pkg.addResource(r); + Exception dpe = Assertions.assertThrows(DataPackageException.class, () -> { + new FilebasedResource("resource-one", files, getBasePath()); + }); + Assertions.assertEquals("Path entries for file-based Resources cannot be absolute", dpe.getMessage()); } - private static Resource buildResource(String relativeInPath) throws URISyntaxException { + @Test + @DisplayName("Test reading Resource data rows as Map, ensuring we get values of " + + "the correct Schema Field type") + public void testReadMapped1() throws Exception{ + String[][] referenceData = new String[][]{ + {"city","year","population"}, + {"london","2017","8780000"}, + {"paris","2017","2240000"}, + {"rome","2017","2860000"}}; + Resource resource = buildResource("/fixtures/data/population.csv"); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/population_schema.json"), true); + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + List> mappedData = resource.getMappedData(false); + Assertions.assertEquals(3, mappedData.size()); + String[] headers = referenceData[0]; + //need to omit the table header in the referenceData + for (int i = 0; i < mappedData.size(); i++) { + String[] refRow = referenceData[i+1]; + Map testData = mappedData.get(i); + // ensure row size is correct + Assertions.assertEquals(refRow.length, testData.size()); + + // ensure we get the headers in the right sort order + ArrayList testDataColKeys = new ArrayList<>(testData.keySet()); + String[] testHeaders = testDataColKeys.toArray(new String[]{}); + Assertions.assertArrayEquals(headers, testHeaders); + + // validate values match and types are as expected + Assertions.assertEquals(refRow[0], testData.get(testDataColKeys.get(0))); //String value for city name + Assertions.assertInstanceOf(Year.class, testData.get(testDataColKeys.get(1))); + Assertions.assertEquals(refRow[1], ((Year)testData.get(testDataColKeys.get(1))).toString());//Year value for year + Assertions.assertInstanceOf(BigInteger.class, testData.get(testDataColKeys.get(2))); //String value for city name + Assertions.assertEquals(refRow[2], testData.get(testDataColKeys.get(2)).toString());//BigInteger value for population + } + } + + @Test + @DisplayName("Test setting invalid 'profile' property, must throw") + public void testSetInvalidProfile() throws Exception { + Resource resource = buildResource("/fixtures/data/population.csv"); + + Assertions.assertThrows(DataPackageValidationException.class, + () -> resource.setProfile(PROFILE_DATA_PACKAGE_DEFAULT)); + Assertions.assertThrows(DataPackageValidationException.class, + () -> resource.setProfile(PROFILE_TABULAR_DATA_PACKAGE)); + Assertions.assertDoesNotThrow(() -> resource.setProfile(PROFILE_DATA_RESOURCE_DEFAULT)); + Assertions.assertDoesNotThrow(() -> resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE)); + } + + @Test + @DisplayName("Read a resource with 3 tables and get data as CSV") + public void testResourceToCsvDataFromMultipartFilePath() throws Exception { + String refStr = "city,location\n" + + "libreville,\"0.41,9.29\"\n" + + "dakar,\"14.71,-17.53\"\n" + + "ouagadougou,\"12.35,-1.67\"\n" + + "barranquilla,\"10.98,-74.88\"\n" + + "rio de janeiro,\"-22.91,-43.72\"\n" + + "cuidad de guatemala,\"14.62,-90.56\"\n" + + "london,\"51.5,-0.11\"\n" + + "paris,\"48.85,2.3\"\n" + + "rome,\"41.89,12.51\""; + + String[] paths = new String[]{ + "data/cities.csv", + "data/cities2.csv", + "data/cities3.csv"}; + List files = new ArrayList<>(); + for (String file : paths) { + files.add(new File(file)); + } + Resource resource = new FilebasedResource("coordinates", files, getBasePath()); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); + Dialect dialect = Dialect.DEFAULT; + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + resource.setDialect(dialect); + + String dataAsCsv = resource.getDataAsCsv(dialect, schema); + Assertions.assertEquals(refStr.replaceAll("[\n\r]+", "\n"), + dataAsCsv.replaceAll("[\n\r]+", "\n")); + } + + + @Test + @DisplayName("Read a resource with 3 tables and get data as Json") + public void testResourceToJsonDataFromMultipartFilePath() throws Exception { + String refStr = "[ {\n" + + " \"city\" : \"libreville\",\n" + + " \"location\" : \"0.41,9.29\"\n" + + "}, {\n" + + " \"city\" : \"dakar\",\n" + + " \"location\" : \"14.71,-17.53\"\n" + + "}, {\n" + + " \"city\" : \"ouagadougou\",\n" + + " \"location\" : \"12.35,-1.67\"\n" + + "}, {\n" + + " \"city\" : \"barranquilla\",\n" + + " \"location\" : \"10.98,-74.88\"\n" + + "}, {\n" + + " \"city\" : \"rio de janeiro\",\n" + + " \"location\" : \"-22.91,-43.72\"\n" + + "}, {\n" + + " \"city\" : \"cuidad de guatemala\",\n" + + " \"location\" : \"14.62,-90.56\"\n" + + "}, {\n" + + " \"city\" : \"london\",\n" + + " \"location\" : \"51.5,-0.11\"\n" + + "}, {\n" + + " \"city\" : \"paris\",\n" + + " \"location\" : \"48.85,2.3\"\n" + + "}, {\n" + + " \"city\" : \"rome\",\n" + + " \"location\" : \"41.89,12.51\"\n" + + "} ]"; + + String[] paths = new String[]{ + "data/cities.csv", + "data/cities2.csv", + "data/cities3.csv"}; + List files = new ArrayList<>(); + for (String file : paths) { + files.add(new File(file)); + } + Resource resource = new FilebasedResource("coordinates", files, getBasePath()); + Schema schema = Schema.fromJson(new File(getTestDataDirectory() + , "/fixtures/schema/city_location_schema.json"), true); + Dialect dialect = Dialect.DEFAULT; + // Set the profile to tabular data resource. + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + resource.setSchema(schema); + resource.setDialect(dialect); + + String dataAsCsv = resource.getDataAsJson(); + Assertions.assertEquals(refStr.replaceAll("[\n\r]+", "\n"), + dataAsCsv.replaceAll("[\n\r]+", "\n")); + } + + private static Resource buildResource(String relativeInPath) throws URISyntaxException { URL sourceFileUrl = ResourceTest.class.getResource(relativeInPath); Path path = Paths.get(sourceFileUrl.toURI()); Path parent = path.getParent(); @@ -547,8 +683,7 @@ private static String getFileContents(String fileName) { } private List getExpectedPopulationData(){ - List expectedData = new ArrayList(); - //expectedData.add(new String[]{"city", "year", "population"}); + List expectedData = new ArrayList<>(); expectedData.add(new String[]{"london", "2017", "8780000"}); expectedData.add(new String[]{"paris", "2017", "2240000"}); expectedData.add(new String[]{"rome", "2017", "2860000"}); @@ -557,7 +692,7 @@ private List getExpectedPopulationData(){ } private List getExpectedAlternatePopulationData(){ - List expectedData = new ArrayList(); + List expectedData = new ArrayList<>(); expectedData.add(new String[]{"2017", "london", "8780000"}); expectedData.add(new String[]{"2017", "paris", "2240000"}); expectedData.add(new String[]{"2017", "rome", "2860000"}); diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java new file mode 100644 index 0000000..a7ae325 --- /dev/null +++ b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java @@ -0,0 +1,254 @@ +package io.frictionlessdata.datapackage.resource; + +import io.frictionlessdata.datapackage.Dialect; +import io.frictionlessdata.datapackage.Package; +import io.frictionlessdata.datapackage.TestUtil; +import io.frictionlessdata.tableschema.schema.Schema; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import org.apache.commons.csv.CSVFormat; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static io.frictionlessdata.datapackage.Profile.PROFILE_TABULAR_DATA_RESOURCE; + +/** + * Ensure datapackages are written in a valid format and can be read back. Compare data to see it matches + */ +public class RoundtripTest { + private static final CSVFormat csvFormat = TableDataSource + .getDefaultCsvFormat().builder().setDelimiter('\t').get(); + + @Test + @DisplayName("Roundtrip test - write datapackage, read again and compare data") + // this lead to the discovery of https://github.com/frictionlessdata/specs/issues/666, just in case + // we see regressions in the specs somehow + public void dogfoodingTest() throws Exception { + // create a new Package, set Schema and data and write out to temp storage + List resources = new ArrayList<>(); + Package pkg = new Package(resources); + + JSONDataResource res = new JSONDataResource("population", resourceContent); + //set a schema to guarantee the ordering of properties + Schema schema = Schema.fromJson( + new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true); + res.setSchema(schema); + res.setShouldSerializeToFile(true); + res.setSerializationFormat(Resource.FORMAT_CSV); + res.setDialect(Dialect.fromCsvFormat(csvFormat)); + pkg.addResource(res); + + Path tempDirPath = Files.createTempDirectory("datapackage-"); + File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); + pkg.write(createdFile, true); + + // create new Package from the serialized form and check they are equal + Package testPkg = new Package(createdFile.toPath(), true); + Assertions.assertEquals(1, testPkg.getResources().size()); + + Resource testRes = testPkg.getResource("population"); + List testData = testRes.getData(false, true, false, false); + Resource validationRes = pkg.getResource("population"); + List validationData = validationRes.getData(false, true, false, false); + Assertions.assertEquals(validationData.size(), testData.size()); + + for (int i = 0; i < validationData.size(); i++) { + Assertions.assertArrayEquals(((Object[])validationData.get(i)), ((Object[])testData.get(i))); + + } + } + + @Test + @DisplayName("Roundtrip resource") + void validateResourceRoundtrip() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json"); + Package dp = new Package(resourcePath, true); + Resource referenceResource = dp.getResource("test2"); + List referenceData = referenceResource.getData(false, false, true, false); + String data = createCSV(referenceResource.getHeaders(), referenceData); + Resource newResource = new CSVDataResource(referenceResource.getName(),data); + newResource.setDescription(referenceResource.getDescription()); + newResource.setSchema(referenceResource.getSchema()); + newResource.setSerializationFormat(Resource.FORMAT_CSV); + List testData = newResource.getData(false, false, true, false); + Assertions.assertArrayEquals(referenceData.toArray(), testData.toArray()); + } + + @Test + @DisplayName("Create data resource, compare descriptor") + void validateCreateResourceDescriptorRoundtrip() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json"); + Package pkg = new Package(resourcePath, true); + JSONDataResource resource = new JSONDataResource("test3", resourceContent); + resource.setSchema(Schema.fromJson( + new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true)); + + resource.setShouldSerializeToFile(false); + resource.setSerializationFormat(Resource.FORMAT_CSV); + resource.setDialect(Dialect.fromCsvFormat(csvFormat)); + resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE); + resource.setEncoding("utf-8"); + pkg.addResource(resource); + Path tempDirPath = Files.createTempDirectory("datapackage-"); + File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); + pkg.write(createdFile, true); + + // create new Package from the serialized form and check they are equal + Package testPkg = new Package(createdFile.toPath(), true); + String json = testPkg.asJson(); + Assertions.assertEquals( + descriptorContentInlined.replaceAll("[\n\r]+", "\n"), + json.replaceAll("[\n\r]+", "\n") + ); + } + + + @Test + @DisplayName("Create data resource and make it write to file, compare descriptor") + void validateCreateResourceDescriptorRoundtrip2() throws Exception { + Path resourcePath = TestUtil.getResourcePath("/fixtures/datapackages/roundtrip/datapackage.json"); + Package pkg = new Package(resourcePath, true); + JSONDataResource resource = new JSONDataResource("test3", resourceContent); + resource.setSchema(Schema.fromJson( + new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true)); + + resource.setShouldSerializeToFile(true); + resource.setSerializationFormat(Resource.FORMAT_CSV); + resource.setDialect(Dialect.fromCsvFormat(csvFormat)); + resource.setProfile(PROFILE_TABULAR_DATA_RESOURCE); + resource.setEncoding("utf-8"); + pkg.addResource(resource); + Path tempDirPath = Files.createTempDirectory("datapackage-"); + File createdFile = new File(tempDirPath.toFile(), "test_save_datapackage.zip"); + pkg.write(createdFile, true); + + // create new Package from the serialized form and check they are equal + Package testPkg = new Package(createdFile.toPath(), true); + String json = testPkg.asJson(); + Assertions.assertEquals( + descriptorContent.replaceAll("[\n\r]+", "\n"), + json.replaceAll("[\n\r]+", "\n") + ); + } + + private static String createCSV(String[] headers, List data) { + StringBuilder sb = new StringBuilder(); + sb.append(String.join(",", Arrays.asList(headers))); + sb.append("\n"); + for (Object[] row : data) { + sb.append(Arrays.stream(row) + .map((o)-> (o == null) ? "" : o.toString()) + .collect(Collectors.joining(","))); + sb.append("\n"); + } + return sb.toString(); + } + + + private static final String resourceContent = "[\n" + + " {\n" + + "\t \"city\": \"london\",\n" + + "\t \"year\": 2017,\n" + + "\t \"population\": 8780000\n" + + "\t},\n" + + "\t{\n" + + "\t \"city\": \"paris\",\n" + + "\t \"year\": 2017,\n" + + "\t \"population\": 2240000\n" + + "\t},\n" + + "\t{\n" + + "\t \"city\": \"rome\",\n" + + "\t \"year\": 2017,\n" + + "\t \"population\": 2860000\n" + + "\t}\n" + + " ]"; + + private static final String descriptorContent = "{\n" + + " \"name\" : \"foreign-keys\",\n" + + " \"profile\" : \"data-package\",\n" + + " \"resources\" : [ {\n" + + " \"name\" : \"test2\",\n" + + " \"profile\" : \"data-resource\",\n" + + " \"schema\" : \"schema/test2.json\",\n" + + " \"path\" : \"data/test2.csv\"\n" + + " }, {\n" + + " \"name\" : \"test3\",\n" + + " \"profile\" : \"tabular-data-resource\",\n" + + " \"encoding\" : \"utf-8\",\n" + + " \"format\" : \"csv\",\n"+ + " \"dialect\" : \"dialect/test3.json\",\n" + + " \"schema\" : \"schema/population_schema.json\",\n" + + " \"path\" : \"data/test3.csv\"\n" + + " } ]\n" + + "}"; + + + private static String descriptorContentInlined ="{\n" + + " \"name\" : \"foreign-keys\",\n" + + " \"profile\" : \"data-package\",\n" + + " \"resources\" : [ {\n" + + " \"name\" : \"test2\",\n" + + " \"profile\" : \"data-resource\",\n" + + " \"schema\" : \"schema/test2.json\",\n" + + " \"path\" : \"data/test2.csv\"\n" + + " }, {\n" + + " \"name\" : \"test3\",\n" + + " \"profile\" : \"tabular-data-resource\",\n" + + " \"encoding\" : \"utf-8\",\n" + + " \"format\" : \"json\",\n" + + " \"dialect\" : {\n" + + " \"caseSensitiveHeader\" : false,\n" + + " \"quoteChar\" : \"\\\"\",\n" + + " \"doubleQuote\" : true,\n" + + " \"delimiter\" : \"\\t\",\n" + + " \"lineTerminator\" : \"\\r\\n\",\n" + + " \"nullSequence\" : \"\",\n" + + " \"header\" : true,\n" + + " \"csvddfVersion\" : 1.2,\n" + + " \"skipInitialSpace\" : true\n" + + " },\n" + + " \"schema\" : {\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"city\",\n" + + " \"title\" : \"city\",\n" + + " \"type\" : \"string\",\n" + + " \"format\" : \"default\",\n" + + " \"description\" : \"The city.\"\n" + + " }, {\n" + + " \"name\" : \"year\",\n" + + " \"title\" : \"year\",\n" + + " \"type\" : \"year\",\n" + + " \"format\" : \"default\",\n" + + " \"description\" : \"The year.\"\n" + + " }, {\n" + + " \"name\" : \"population\",\n" + + " \"title\" : \"population\",\n" + + " \"type\" : \"integer\",\n" + + " \"format\" : \"default\",\n" + + " \"description\" : \"The population.\"\n" + + " } ]\n" + + " },\n" + + " \"data\" : [ {\n" + + " \"city\" : \"london\",\n" + + " \"year\" : 2017,\n" + + " \"population\" : 8780000\n" + + " }, {\n" + + " \"city\" : \"paris\",\n" + + " \"year\" : 2017,\n" + + " \"population\" : 2240000\n" + + " }, {\n" + + " \"city\" : \"rome\",\n" + + " \"year\" : 2017,\n" + + " \"population\" : 2860000\n" + + " } ]\n" + + " } ]\n" + + "}"; +} diff --git a/src/test/resources/fixtures/datapackages/bean-iterator/data/employees.csv b/src/test/resources/fixtures/datapackages/bean-iterator/data/employees.csv new file mode 100644 index 0000000..ad6a1f1 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/bean-iterator/data/employees.csv @@ -0,0 +1,4 @@ +id,name,dateOfBirth,isAdmin,addressCoordinates,contractLength,info +1,John Doe,1976-01-13,true,"{""lon"": 90, ""lat"": 45}",P2DT3H4M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +2,Frank McKrank,1992-02-14,false,"{""lon"": 90, ""lat"": 45}",PT15M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +3,Pencil Vester,1983-03-16,false,"{""lon"": 90, ""lat"": 45}",PT20.345S,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/bean-iterator/datapackage.json b/src/test/resources/fixtures/datapackages/bean-iterator/datapackage.json new file mode 100644 index 0000000..f7baab7 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/bean-iterator/datapackage.json @@ -0,0 +1,11 @@ +{ + "name": "employees", + "profile": "tabular-data-package", + "resources": [{ + "name": "employee-data", + "path": "data/employees.csv", + "schema": "employee_schema.json", + "profile": "tabular-data-resource" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/bean-iterator/employee_schema.json b/src/test/resources/fixtures/datapackages/bean-iterator/employee_schema.json new file mode 100644 index 0000000..2df5efc --- /dev/null +++ b/src/test/resources/fixtures/datapackages/bean-iterator/employee_schema.json @@ -0,0 +1,41 @@ +{ + "fields":[ + { + "name":"id", + "format":"default", + "type":"integer" + }, + { + "name":"name", + "format":"default", + "type":"string" + }, + { + "name":"dateOfBirth", + "format":"default", + "type":"date" + }, + { + "name":"isAdmin", + "format":"default", + "type":"boolean", + "trueValues": ["T"], + "falseValues": ["F"] + }, + { + "name":"addressCoordinates", + "format":"object", + "type":"geopoint" + }, + { + "name":"contractLength", + "format":"default", + "type":"duration" + }, + { + "name":"info", + "format":"default", + "type":"object" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/constraint-violation/data/person.csv b/src/test/resources/fixtures/datapackages/constraint-violation/data/person.csv new file mode 100644 index 0000000..4ba75a0 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/constraint-violation/data/person.csv @@ -0,0 +1,5 @@ +firstname,lastname,gender,age +John,Doe,male,30 +Jane,Smith,female,25 +Alice,Johnson,female,19 +Bob,Williams,male,1 \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/constraint-violation/datapackage.json b/src/test/resources/fixtures/datapackages/constraint-violation/datapackage.json new file mode 100644 index 0000000..500d237 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/constraint-violation/datapackage.json @@ -0,0 +1,52 @@ +{ + "name":"csv-validation-using-ig", + "description":"Validates Person", + "dialect":{ + "delimiter":"," + }, + "resources":[ + { + "name":"person_data", + "path":"datapackages/constraint-violation/data/person.csv", + "schema":{ + "fields":[ + { + "name":"firstname", + "type":"string", + "description":"The first name of the person.", + "constraints":{ + "required":true + } + }, + { + "name":"lastname", + "type":"string", + "description":"The last name of the person.", + "constraints":{ + "required":true + } + }, + { + "name":"gender", + "type":"string", + "description":"Gender of the person. Valid values are 'male' or 'female'.", + "constraints":{ + "enum":[ + "male", + "female" + ] + } + }, + { + "name":"age", + "type":"integer", + "description":"The age of the person. Must be greater than 18.", + "constraints":{ + "minimum":19 + } + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_arrays.json b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_arrays.json new file mode 100644 index 0000000..8ead56c --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_arrays.json @@ -0,0 +1,6 @@ +[ + ["id", "name", "city"], + ["1", "Arsenal", "London"], + ["2", "Real", "Madrid"], + ["3", "Bayern", "Munich"] +] \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_no_headers.csv b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_no_headers.csv new file mode 100644 index 0000000..1b04acc --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_no_headers.csv @@ -0,0 +1,3 @@ +1, Arsenal, London +2, Real, Madrid +3, Bayern, Munich \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_objects.json b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_objects.json new file mode 100644 index 0000000..3a9dd5e --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_objects.json @@ -0,0 +1,17 @@ +[ + { + "id":1, + "name":"Arsenal", + "city":"London" + }, + { + "id":2, + "name":"Real", + "city":"Madrid" + }, + { + "id":3, + "name":"Bayern", + "city":"Munich" + } +] \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_with_headers.csv b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_with_headers.csv new file mode 100644 index 0000000..31e7e01 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/data/teams_with_headers.csv @@ -0,0 +1,4 @@ +id, name, city +1, Arsenal, London +2, Real, Madrid +3, Bayern, Munich \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/datapackage.json b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/datapackage.json new file mode 100644 index 0000000..d42fc9d --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-data-formats_incl_invalid/datapackage.json @@ -0,0 +1,272 @@ +{ + "name": "different-data-formats", + "profile":"tabular-data-package", + "resources": [ + { + "name": "teams_with_headers_csv_file_with_schema", + "comment": "File-based CSV data, with headers and schema, this will read OK and return 3 rows", + "path": "data/teams_with_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_with_headers_csv_file_no_schema", + "comment": "File-based CSV data, with headers", + "path": "data/teams_with_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_no_headers_csv_file_with_schema", + "comment": "File-based CSV data, no headers, with Schema, this will lead an error on read, as Schema and first row do not match", + "path": "data/teams_no_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_no_headers_csv_file_no_schema", + "comment": "File-based CSV data, no headers, this will lead to the first row being dropped", + "path": "data/teams_no_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_no_headers_inline_csv_with_schema", + "comment": "Inline CSV data, no headers, with Schema, this will lead an error on read, as Schema and first row do not match", + "data": "1, Arsenal, London\n2, Real, Madrid\n3, Bayern, Munich", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_arrays_no_headers_inline_with_schema", + "comment": "Inline JSON array data, no headers, with Schema, this will lead an error on read, as Schema and first row do not match", + "data": [ + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_arrays_inline_with_headers_with_schema", + "comment": "Inline JSON array data, with headers, this will read OK and return 3 rows", + "data": [ + ["id", "name", "city"], + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_arrays_inline_with_headers_no_schema", + "comment": "Inline JSON array data, with headers and no Schema, this will read OK and return 3 string rows", + "data": [ + ["id", "name", "city"], + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource" + }, + { + "name": "teams_no_headers_inline_csv_no_schema", + "comment": "Inline CSV data, no headers, no Schema, this will read OK and return 2 string rows", + "data": "1, Arsenal, London\n2, Real, Madrid\n3, Bayern, Munich", + "format": "csv", + "profile": "tabular-data-resource" + }, + { + "name": "teams_arrays_file_with_headers_with_schema", + "comment": "File-based JSON array data, with headers. This is not strictly supported according to the spec, this will read OK and return 3 rows", + "path": "data/teams_arrays.json", + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_objects_inline_with_schema", + "comment": "Inline JSON object data, with Schema no headers needed, this will read OK and return 3 rows", + "data": [ + { + "id":1, + "name":"Arsenal", + "city":"London" + }, + { + "id":2, + "name":"Real", + "city":"Madrid" + }, + { + "id":3, + "name":"Bayern", + "city":"Munich" + } + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_objects_file_with_schema", + "comment": "File-based JSON object data, no headers needed. This is not strictly supported according to the spec, this will read OK and return 3 rows", + "path": "data/teams_objects.json", + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_arrays.json b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_arrays.json new file mode 100644 index 0000000..8ead56c --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_arrays.json @@ -0,0 +1,6 @@ +[ + ["id", "name", "city"], + ["1", "Arsenal", "London"], + ["2", "Real", "Madrid"], + ["3", "Bayern", "Munich"] +] \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_no_headers.csv b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_no_headers.csv new file mode 100644 index 0000000..1b04acc --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_no_headers.csv @@ -0,0 +1,3 @@ +1, Arsenal, London +2, Real, Madrid +3, Bayern, Munich \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_objects.json b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_objects.json new file mode 100644 index 0000000..3a9dd5e --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_objects.json @@ -0,0 +1,17 @@ +[ + { + "id":1, + "name":"Arsenal", + "city":"London" + }, + { + "id":2, + "name":"Real", + "city":"Madrid" + }, + { + "id":3, + "name":"Bayern", + "city":"Munich" + } +] \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_with_headers.csv b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_with_headers.csv new file mode 100644 index 0000000..31e7e01 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-valid-data-formats/data/teams_with_headers.csv @@ -0,0 +1,4 @@ +id, name, city +1, Arsenal, London +2, Real, Madrid +3, Bayern, Munich \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/different-valid-data-formats/datapackage.json b/src/test/resources/fixtures/datapackages/different-valid-data-formats/datapackage.json new file mode 100644 index 0000000..8cf4f0c --- /dev/null +++ b/src/test/resources/fixtures/datapackages/different-valid-data-formats/datapackage.json @@ -0,0 +1,262 @@ +{ + "name": "different-data-formats", + "profile":"tabular-data-package", + "resources": [ + { + "name": "teams_with_headers_csv_file", + "comment": "File-based CSV data, with headers", + "path": "data/teams_with_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_no_headers_csv_file_with_schema", + "comment": "File-based CSV data, no headers, this will lead to an error as Schema and first row do not match", + "path": "data/teams_no_headers.csv", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_no_headers_inline_csv_with_schema", + "comment": "Inline CSV data, no headers, this will lead an error, as Schema and first row do not match", + "data": "1, Arsenal, London\n2, Real, Madrid\n3, Bayern, Munich", + "format": "csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_arrays_headers_no_schema_inline", + "comment": "Inline JSON array data, headers and no schema, this will lead to the three rows of data", + "data": [ + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_arrays_no_headers_inline_with_schema", + "comment": "Inline JSON array data, no headers, this will lead to an error as Schema and first row do not match", + "data": [ + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + }, + "dialect": { + "delimiter": ",", + "doubleQuote": true + } + }, + { + "name": "teams_arrays_inline", + "comment": "Inline JSON array data, with headers", + "data": [ + ["id", "name", "city"], + [1, "Arsenal", "London"], + [2, "Real", "Madrid"], + [3, "Bayern", "Munich"] + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_arrays_file", + "comment": "File-based JSON array data, with headers. This is not strictly supported according to the spec", + "path": "data/teams_arrays.json", + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_objects_inline", + "comment": "Inline JSON object data, no headers needed", + "data": [ + { + "id":1, + "name":"Arsenal", + "city":"London" + }, + { + "id":2, + "name":"Real", + "city":"Madrid" + }, + { + "id":3, + "name":"Bayern", + "city":"Munich" + } + ], + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + }, + { + "name": "teams_objects_file", + "comment": "File-based JSON object data, no headers needed. This is not strictly supported according to the spec", + "path": "data/teams_objects.json", + "format": "json", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/employees/data/employees.csv b/src/test/resources/fixtures/datapackages/employees/data/employees.csv index 8f65a33..ad6a1f1 100644 --- a/src/test/resources/fixtures/datapackages/employees/data/employees.csv +++ b/src/test/resources/fixtures/datapackages/employees/data/employees.csv @@ -1,4 +1,4 @@ id,name,dateOfBirth,isAdmin,addressCoordinates,contractLength,info -1,John Doe,1976-01-13,T,"{""lon"": 90, ""lat"": 45}",P2DT3H4M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" -2,Frank McKrank,1992-02-14,F,"{""lon"": 90, ""lat"": 45}",PT15M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" -3,Pencil Vester,1983-03-16,F,"{""lon"": 90, ""lat"": 45}",PT20.345S,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" \ No newline at end of file +1,John Doe,1976-01-13,true,"{""lon"": 90, ""lat"": 45}",P2DT3H4M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +2,Frank McKrank,1992-02-14,false,"{""lon"": 90, ""lat"": 45}",PT15M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +3,Pencil Vester,1983-03-16,false,"{""lon"": 90, ""lat"": 45}",PT20.345S,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/employees/datapackage.json b/src/test/resources/fixtures/datapackages/employees/datapackage.json index f7baab7..ccb8588 100644 --- a/src/test/resources/fixtures/datapackages/employees/datapackage.json +++ b/src/test/resources/fixtures/datapackages/employees/datapackage.json @@ -4,7 +4,7 @@ "resources": [{ "name": "employee-data", "path": "data/employees.csv", - "schema": "employee_schema.json", + "schema": "schema.json", "profile": "tabular-data-resource" } ] diff --git a/src/test/resources/fixtures/datapackages/employees/employee_schema.json b/src/test/resources/fixtures/datapackages/employees/schema.json similarity index 100% rename from src/test/resources/fixtures/datapackages/employees/employee_schema.json rename to src/test/resources/fixtures/datapackages/employees/schema.json diff --git a/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/data/employees.csv b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/data/employees.csv new file mode 100644 index 0000000..ad6a1f1 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/data/employees.csv @@ -0,0 +1,4 @@ +id,name,dateOfBirth,isAdmin,addressCoordinates,contractLength,info +1,John Doe,1976-01-13,true,"{""lon"": 90, ""lat"": 45}",P2DT3H4M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +2,Frank McKrank,1992-02-14,false,"{""lon"": 90, ""lat"": 45}",PT15M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +3,Pencil Vester,1983-03-16,false,"{""lon"": 90, ""lat"": 45}",PT20.345S,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/datapackage.json b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/datapackage.json new file mode 100644 index 0000000..f7baab7 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/datapackage.json @@ -0,0 +1,11 @@ +{ + "name": "employees", + "profile": "tabular-data-package", + "resources": [{ + "name": "employee-data", + "path": "data/employees.csv", + "schema": "employee_schema.json", + "profile": "tabular-data-resource" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/employee_schema.json b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/employee_schema.json new file mode 100644 index 0000000..2df5efc --- /dev/null +++ b/src/test/resources/fixtures/datapackages/employees_scheme_wont_match_truevalues/employee_schema.json @@ -0,0 +1,41 @@ +{ + "fields":[ + { + "name":"id", + "format":"default", + "type":"integer" + }, + { + "name":"name", + "format":"default", + "type":"string" + }, + { + "name":"dateOfBirth", + "format":"default", + "type":"date" + }, + { + "name":"isAdmin", + "format":"default", + "type":"boolean", + "trueValues": ["T"], + "falseValues": ["F"] + }, + { + "name":"addressCoordinates", + "format":"object", + "type":"geopoint" + }, + { + "name":"contractLength", + "format":"default", + "type":"duration" + }, + { + "name":"info", + "format":"default", + "type":"object" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/foreign_keys_invalid.json b/src/test/resources/fixtures/datapackages/foreign_keys_invalid.json new file mode 100644 index 0000000..8f2b7d0 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/foreign_keys_invalid.json @@ -0,0 +1,47 @@ +{ + "name": "foreign-keys", + "resources": [ + { + "name": "teams", + "data": [ + ["id", "name", "city"], + ["1", "Arsenal", "London"], + ["2", "Real", "Madrid"], + ["3", "Bayern", "Munich"] + ], + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ], + "foreignKeys": [ + { + "fields": "city", + "reference": { + "resource": "cities", + "fields": "name" + } + } + ] + } + }, + { + "name": "cities", + "data": [ + ["name", "country"], + ["London", "England"], + ["Madrid", "Spain"] + ] + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/foreign_keys_valid.json b/src/test/resources/fixtures/datapackages/foreign_keys_valid.json new file mode 100644 index 0000000..c3fab05 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/foreign_keys_valid.json @@ -0,0 +1,81 @@ +{ + "name": "foreign-keys", + "resources": [ + { + "name": "teams", + "data": [ + [ + "id", + "name", + "city" + ], + [ + "1", + "Arsenal", + "London" + ], + [ + "2", + "Real", + "Madrid" + ], + [ + "3", + "Bayern", + "Munich" + ], + [ + "4", + "MadeUpFC", + null + ] + ], + "schema": { + "fields": [ + { + "name": "id", + "type": "integer" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "city", + "type": "string" + } + ], + "foreignKeys": [ + { + "fields": "city", + "reference": { + "resource": "cities", + "fields": "name" + } + } + ] + } + }, + { + "name": "cities", + "data": [ + [ + "name", + "country" + ], + [ + "Munich", + "Germany" + ], + [ + "London", + "England" + ], + [ + "Madrid", + "Spain" + ] + ] + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/multi-data/datapackage.json b/src/test/resources/fixtures/datapackages/multi-data/datapackage.json index bbbf9dd..d76fee4 100644 --- a/src/test/resources/fixtures/datapackages/multi-data/datapackage.json +++ b/src/test/resources/fixtures/datapackages/multi-data/datapackage.json @@ -2,10 +2,12 @@ "name": "multi-data", "resources": [{ "name": "first-resource", + "profile": "tabular-data-resource", "path": ["data/cities.csv", "data/cities2.csv", "data/cities3.csv"] }, { "name": "second-resource", + "profile": "tabular-data-resource", "path": [ "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities.csv", "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/cities2.csv", @@ -13,17 +15,20 @@ ] }, { - "name": "third-resource", + "name": "third-resource", + "profile": "tabular-data-resource", "schema": "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/schema/population_schema.json", "path": "data/population.csv" }, { - "name": "fourth-resource", + "name": "fourth-resource", + "profile": "tabular-data-resource", "schema": "schema/population_schema.json", "path": "https://raw.githubusercontent.com/frictionlessdata/datapackage-java/master/src/test/resources/fixtures/data/population.csv" }, { - "name": "fifth-resource", + "name": "fifth-resource", + "profile": "tabular-data-resource", "schema": "schema/population_schema.json", "path": "data/population.csv", "dialect": "dialect.json" diff --git a/src/test/resources/fixtures/datapackages/non-tabular/data/frictionless-color-full-logo.svg b/src/test/resources/fixtures/datapackages/non-tabular/data/frictionless-color-full-logo.svg new file mode 100644 index 0000000..73a37a4 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/non-tabular/data/frictionless-color-full-logo.svg @@ -0,0 +1,23 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/non-tabular/datapackage.json b/src/test/resources/fixtures/datapackages/non-tabular/datapackage.json new file mode 100644 index 0000000..ef4c3a6 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/non-tabular/datapackage.json @@ -0,0 +1,14 @@ +{ + "name": "non-tabular", + "creator": "Horst", + "profile": "data-package", + "resources": [ + { + "path": "data/frictionless-color-full-logo.svg", + "name": "logo-svg", + "format": "svg", + "mediatype": "image/svg+xml", + "profile": "data-resource" + } + ] +} diff --git a/src/test/resources/fixtures/datapackages/roundtrip/data/test2.csv b/src/test/resources/fixtures/datapackages/roundtrip/data/test2.csv new file mode 100644 index 0000000..14a53d6 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/roundtrip/data/test2.csv @@ -0,0 +1,15 @@ +cola,colb,colc,cold,cole,colf +substance1,,SUB1,123,, +substance1,,SUB1,123,, +substance2,,,,, +substance3,,,345,, +substance1,,SUB1,123,, +substance4,,,678,, +substance5,,SUB5,890,, +substance6,,,365,, +substance7,,,143,, +substance5,,,326,, +substance1,,SUB1,123,, +substance8,,,9837,, +substance9,,SUB9,835,, +substance10,,,473,, diff --git a/src/test/resources/fixtures/datapackages/roundtrip/datapackage.json b/src/test/resources/fixtures/datapackages/roundtrip/datapackage.json new file mode 100644 index 0000000..2486f0e --- /dev/null +++ b/src/test/resources/fixtures/datapackages/roundtrip/datapackage.json @@ -0,0 +1,43 @@ +{ + "name": "foreign-keys", + "resources": [ + { + "name": "test2", + "path": "data/test2.csv", + "schema": { + "fields": [ + { + "name":"cola", + "type":"string", + "format":"default" + }, + { + "name":"colb", + "type":"string", + "format":"default" + }, + { + "name":"colc", + "type":"string", + "format":"default" + }, + { + "name":"cold", + "type":"integer", + "format":"default" + }, + { + "name":"cole", + "type":"string", + "format":"default" + }, + { + "name":"colf", + "type":"string", + "format":"default" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/with-image/data/employees.csv b/src/test/resources/fixtures/datapackages/with-image/data/employees.csv new file mode 100644 index 0000000..ad6a1f1 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/with-image/data/employees.csv @@ -0,0 +1,4 @@ +id,name,dateOfBirth,isAdmin,addressCoordinates,contractLength,info +1,John Doe,1976-01-13,true,"{""lon"": 90, ""lat"": 45}",P2DT3H4M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +2,Frank McKrank,1992-02-14,false,"{""lon"": 90, ""lat"": 45}",PT15M,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" +3,Pencil Vester,1983-03-16,false,"{""lon"": 90, ""lat"": 45}",PT20.345S,"{""ssn"": 90, ""pin"": 45, ""rate"": 83.23}" \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/with-image/datapackage.json b/src/test/resources/fixtures/datapackages/with-image/datapackage.json new file mode 100644 index 0000000..a1c4e88 --- /dev/null +++ b/src/test/resources/fixtures/datapackages/with-image/datapackage.json @@ -0,0 +1,12 @@ +{ + "name": "employees", + "profile": "tabular-data-package", + "image" : "test.png", + "resources": [{ + "name": "employee-data", + "path": "data/employees.csv", + "schema": "schema.json", + "profile": "tabular-data-resource" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/with-image/schema.json b/src/test/resources/fixtures/datapackages/with-image/schema.json new file mode 100644 index 0000000..f34d4ee --- /dev/null +++ b/src/test/resources/fixtures/datapackages/with-image/schema.json @@ -0,0 +1,39 @@ +{ + "fields":[ + { + "name":"id", + "format":"default", + "type":"integer" + }, + { + "name":"name", + "format":"default", + "type":"string" + }, + { + "name":"dateOfBirth", + "format":"default", + "type":"date" + }, + { + "name":"isAdmin", + "format":"default", + "type":"boolean" + }, + { + "name":"addressCoordinates", + "format":"object", + "type":"geopoint" + }, + { + "name":"contractLength", + "format":"default", + "type":"duration" + }, + { + "name":"info", + "format":"default", + "type":"object" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/datapackages/with-image/test.png b/src/test/resources/fixtures/datapackages/with-image/test.png new file mode 100644 index 0000000..2982392 Binary files /dev/null and b/src/test/resources/fixtures/datapackages/with-image/test.png differ diff --git a/src/test/resources/fixtures/files/frictionless-color-full-logo.svg b/src/test/resources/fixtures/files/frictionless-color-full-logo.svg new file mode 100644 index 0000000..73a37a4 --- /dev/null +++ b/src/test/resources/fixtures/files/frictionless-color-full-logo.svg @@ -0,0 +1,23 @@ + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/fixtures/files/sample.pdf b/src/test/resources/fixtures/files/sample.pdf new file mode 100644 index 0000000..6f79ff0 Binary files /dev/null and b/src/test/resources/fixtures/files/sample.pdf differ diff --git a/src/test/resources/fixtures/files/test.png b/src/test/resources/fixtures/files/test.png new file mode 100644 index 0000000..2982392 Binary files /dev/null and b/src/test/resources/fixtures/files/test.png differ diff --git a/src/test/resources/fixtures/full_spec_datapackage.json b/src/test/resources/fixtures/full_spec_datapackage.json new file mode 100644 index 0000000..2f757c4 --- /dev/null +++ b/src/test/resources/fixtures/full_spec_datapackage.json @@ -0,0 +1,50 @@ +{ + "id": "9e2429be-a43e-4051-aab5-981eb27fe2e8", + "name": "world-full", + "title": "world population data", + "profile": "tabular-data-package", + "licenses": [ + { + "name": "ODC-PDDL-1.0", + "path": "http://opendatacommons.org/licenses/pddl/", + "title": "Open Data Commons Public Domain Dedication and License v1.0" + } + ], + "description": "A datapackage for world population data, featuring all fields from https://specs.frictionlessdata.io/data-package/#language", + "homepage": "https://example.com/world-population-data", + "version": "1.0.1", + "sources": [{ + "title": "World Bank and OECD", + "path": "http://data.worldbank.org/indicator/NY.GDP.MKTP.CD" + }], + "contributors": [{ + "title": "Joe Bloggs", + "email": "joe@bloggs.com", + "path": "https://www.bloggs.com", + "role": "author" + }, + { + "title": "Jim Beam", + "email": "jim@example.com", + "path": "https://www.example.com", + "role": "wrangler", + "organization": "Example Corp" + }], + "keywords": ["world", "population", "world bank"], + "image": "https://github.com/frictionlessdata/datapackage-java/tree/main/src/test/resources/fixtures/datapackages/with-image/test.png", + "created": "1985-04-12T23:20:50.52Z", + "resources": [ + { + "name": "population", + "path": "data/population.csv", + "profile":"tabular-data-resource", + "schema": { + "fields": [ + {"name": "city", "type": "string"}, + {"name": "year", "type": "integer"}, + {"name": "population", "type": "integer"} + ] + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/fixtures/multi_data_datapackage2.json b/src/test/resources/fixtures/multi_data_datapackage2.json index e9c5a20..a413506 100644 --- a/src/test/resources/fixtures/multi_data_datapackage2.json +++ b/src/test/resources/fixtures/multi_data_datapackage2.json @@ -1,13 +1,14 @@ { "name": "multi-data", "profile": "tabular-data-package", + "homepage" : "", "resources": [ { "name": "fifth-resource", "profile": "tabular-data-resource", - "schema": "src/test/resources/fixtures/schema/population_schema.json", - "path": "src/test/resources/fixtures/data/population.csv", - "dialect": "src/test/resources/fixtures/dialect.json" + "schema": "schema/population_schema.json", + "path": "data/population.csv", + "dialect": "dialect.json" } ] } \ No newline at end of file diff --git a/src/test/resources/fixtures/multi_data_datapackage_with_invalid_resource_schema.json b/src/test/resources/fixtures/multi_data_datapackage_with_invalid_resource_schema.json index 92f9d21..6d574bf 100644 --- a/src/test/resources/fixtures/multi_data_datapackage_with_invalid_resource_schema.json +++ b/src/test/resources/fixtures/multi_data_datapackage_with_invalid_resource_schema.json @@ -3,12 +3,12 @@ "name": "multi-data", "resources": [{ "name": "first-resource", - "path": ["src/test/resources/fixtures/data/cities.csv", "src/test/resources/fixtures/data/cities2.csv", "src/test/resources/fixtures/data/cities3.csv"] + "path": ["data/cities.csv", "data/cities2.csv", "data/cities3.csv"] }, { "name": "second-resource", - "schema": "src/test/resources/fixtures/schema/invalid_population_schema.json", - "path": "src/test/resources/fixtures/data/population.csv" + "schema": "schema/invalid_population_schema.json", + "path": "data/population.csv" } ] } \ No newline at end of file diff --git a/src/test/resources/fixtures/schema/city_location_schema.json b/src/test/resources/fixtures/schema/city_location_schema.json new file mode 100644 index 0000000..6f1f0a7 --- /dev/null +++ b/src/test/resources/fixtures/schema/city_location_schema.json @@ -0,0 +1,17 @@ +{ + "fields": [ + { + "name": "city", + "format": "default", + "description": "The city.", + "type": "string", + "title": "city" + }, + { + "name": "location", + "description": "The geographic location.", + "type": "geopoint", + "title": "location" + } + ] +} diff --git a/src/test/resources/fixtures/zip/countries-and-currencies.zap b/src/test/resources/fixtures/zip/countries-and-currencies.zap new file mode 100644 index 0000000..34b5027 Binary files /dev/null and b/src/test/resources/fixtures/zip/countries-and-currencies.zap differ diff --git a/src/test/resources/fixtures/zip/non-tabular.zip b/src/test/resources/fixtures/zip/non-tabular.zip new file mode 100644 index 0000000..72f1c74 Binary files /dev/null and b/src/test/resources/fixtures/zip/non-tabular.zip differ