diff --git a/pom.xml b/pom.xml index 18bba50..ef26b73 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 io.frictionlessdata datapackage-java - 0.6.9-SNAPSHOT + 0.7.0-SNAPSHOT jar https://github.com/frictionlessdata/datapackage-java/issues @@ -22,9 +22,9 @@ 8 ${java.version} ${java.version} - 0.6.9 + 0.7.0 1.3 - 5.9.1 + 5.9.2 2.0.5 4.4 3.10.1 @@ -235,14 +235,14 @@ tableschema-java ${tableschema-java-version} - + @@ -252,11 +252,5 @@ test - - org.junit.vintage - junit-vintage-engine - ${junit.version} - test - \ No newline at end of file diff --git a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java index a24dee3..7a3a702 100644 --- a/src/main/java/io/frictionlessdata/datapackage/JSONBase.java +++ b/src/main/java/io/frictionlessdata/datapackage/JSONBase.java @@ -16,9 +16,11 @@ import io.frictionlessdata.tableschema.io.URLFileReference; import io.frictionlessdata.tableschema.schema.Schema; import io.frictionlessdata.tableschema.util.JsonUtil; +import org.apache.commons.lang3.StringUtils; import java.io.*; import java.net.URL; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -188,7 +190,7 @@ public Map getOriginalReferences() { return originalReferences; } - public static Schema buildSchema(JsonNode resourceJson, Object basePath, boolean isArchivePackage) + public static Schema buildSchema(JsonNode resourceJson, Object basePath, Charset encoding, boolean isArchivePackage) throws Exception { FileReference ref = referenceFromJson(resourceJson, JSON_KEY_SCHEMA, basePath); if (null != ref) { @@ -199,7 +201,13 @@ public static Schema buildSchema(JsonNode resourceJson, Object basePath, boolean : null; if (null == schemaObj) return null; - return Schema.fromJson(dereference(schemaObj, basePath, isArchivePackage).toString(), true); + return Schema.fromJson(dereference(schemaObj, basePath, encoding, isArchivePackage).toString(), true); + } + + + public static Schema buildSchema(JsonNode resourceJson, Object basePath, boolean isArchivePackage) + throws Exception { + return buildSchema(resourceJson, basePath, StandardCharsets.UTF_8, isArchivePackage); } public static Dialect buildDialect (JsonNode resourceJson, Object basePath, boolean isArchivePackage) @@ -213,7 +221,7 @@ public static Dialect buildDialect (JsonNode resourceJson, Object basePath, bool : null; if (null == dialectObj) return null; - return Dialect.fromJson(dereference(dialectObj, basePath, isArchivePackage).toString()); + return Dialect.fromJson(dereference(dialectObj, basePath, StandardCharsets.UTF_8, isArchivePackage).toString()); } private static FileReference referenceFromJson(JsonNode resourceJson, String key, Object basePath) @@ -276,9 +284,9 @@ private static String textValueOrNull(JsonNode source, String fieldName) { } - static String getFileContentAsString(InputStream stream) { + static String getFileContentAsString(InputStream stream, Charset cs) { try (BufferedReader rdr = new BufferedReader( - new InputStreamReader(stream, StandardCharsets.UTF_8))) { + new InputStreamReader(stream, cs))) { List lines = rdr .lines() .collect(Collectors.toList()); @@ -288,9 +296,9 @@ static String getFileContentAsString(InputStream stream) { } } - static String getFileContentAsString(URL url) { + static String getFileContentAsString(URL url, Charset cs) { try { - return getFileContentAsString(url.openStream()); + return getFileContentAsString(url.openStream(), cs); } catch (Exception ex) { if (ex instanceof FileNotFoundException) throw new DataPackageValidationException(ex.getMessage(), ex); @@ -298,9 +306,9 @@ static String getFileContentAsString(URL url) { } } - static String getFileContentAsString(File file) { + static String getFileContentAsString(File file, Charset cs) { try (InputStream is = new FileInputStream(file)){ - return getFileContentAsString(is); + return getFileContentAsString(is, cs); } catch (Exception ex) { if ((ex instanceof NoSuchFileException @@ -311,9 +319,9 @@ static String getFileContentAsString(File file) { } } - static String getFileContentAsString(Path filePath) { + static String getFileContentAsString(Path filePath, Charset encoding) { try (InputStream is = Files.newInputStream(filePath)){ - return getFileContentAsString(is); + return getFileContentAsString(is, encoding); } catch (Exception ex) { if ((ex instanceof NoSuchFileException || (ex instanceof FileNotFoundException))) { @@ -349,7 +357,10 @@ private static ZipEntry findZipEntry(ZipFile zipFile, String fileName) { return null; } - protected static String getZipFileContentAsString(Path inFilePath, String fileName) throws IOException { + protected static String getZipFileContentAsString( + Path inFilePath, + String fileName, + Charset encoding) throws IOException { // Read in memory the file inside the zip. ZipFile zipFile = new ZipFile(inFilePath.toFile()); ZipEntry entry = findZipEntry(zipFile, fileName); @@ -362,20 +373,24 @@ protected static String getZipFileContentAsString(Path inFilePath, String fileNa String content; // Read the datapackage.json file inside the zip try (InputStream stream = zipFile.getInputStream(entry)) { - content = getFileContentAsString(stream); + content = getFileContentAsString(stream, encoding); } zipFile.close(); return content; } - public static ObjectNode dereference(File fileObj, Path basePath, boolean isArchivePackage) throws IOException { + public static ObjectNode dereference( + File fileObj, + Path basePath, + Charset encoding, + boolean isArchivePackage) throws IOException { String jsonContentString; if (isArchivePackage) { String filePath = fileObj.getPath(); if (File.separator.equals("\\")) { filePath = filePath.replaceAll("\\\\", "/"); } - jsonContentString = getZipFileContentAsString(basePath, filePath); + jsonContentString = getZipFileContentAsString(basePath, filePath, encoding); } else { /* If reference is file path. from the spec: "SECURITY: / (absolute path) and ../ (relative parent path) @@ -387,7 +402,7 @@ public static ObjectNode dereference(File fileObj, Path basePath, boolean isArch Path securePath = Resource.toSecure(fileObj.toPath(), basePath); if (securePath.toFile().exists()) { // Create the dereferenced schema object from the local file. - jsonContentString = getFileContentAsString(securePath.toFile()); + jsonContentString = getFileContentAsString(securePath.toFile(), encoding); } else { throw new DataPackageFileOrUrlNotFoundException("Local file not found: " + fileObj); } @@ -405,17 +420,17 @@ public static ObjectNode dereference(File fileObj, Path basePath, boolean isArch * @throws IOException if fetching the contents of the URL goes wrong */ - private static ObjectNode dereference(String url, URL basePath) throws IOException { + private static ObjectNode dereference(String url, URL basePath, Charset encoding) throws IOException { JsonNode dereferencedObj; if (isValidUrl(url)) { // Create the dereferenced object from the remote file. - String jsonContentString = getFileContentAsString(new URL(url)); + String jsonContentString = getFileContentAsString(new URL(url), encoding); dereferencedObj = createNode(jsonContentString); } else { URL lURL = new URL(basePath.toExternalForm()+url); if (isValidUrl(lURL)) { - String jsonContentString = getFileContentAsString(lURL); + String jsonContentString = getFileContentAsString(lURL, encoding); dereferencedObj = createNode(jsonContentString); } else { throw new DataPackageFileOrUrlNotFoundException("URL not found"+lURL); @@ -424,7 +439,11 @@ private static ObjectNode dereference(String url, URL basePath) throws IOExcepti return (ObjectNode) dereferencedObj; } - public static ObjectNode dereference(Object obj, Object basePath, boolean isArchivePackage) throws IOException { + public static ObjectNode dereference + (Object obj, + Object basePath, + Charset encoding, + boolean isArchivePackage) throws IOException { if (null == obj) return null; // Object is already a dereferenced object. @@ -432,22 +451,22 @@ else if(obj instanceof ObjectNode){ // Don't need to do anything, just cast and return. return (ObjectNode)obj; } else if (obj instanceof TextNode) { - return dereference(((TextNode) obj).asText(), basePath, isArchivePackage); + return dereference(((TextNode) obj).asText(), basePath, encoding, isArchivePackage); } else if(obj instanceof String){ String reference = (String)obj; if (isValidUrl(reference)) if (basePath instanceof File) { - String jsonString = getFileContentAsString(new URL(reference)); + String jsonString = getFileContentAsString(new URL(reference), encoding); return (ObjectNode) createNode(jsonString); } else { - String jsonString = getFileContentAsString(new URL(reference)); + String jsonString = getFileContentAsString(new URL(reference), encoding); return (ObjectNode) createNode(jsonString); } else if (basePath instanceof URL) { - return dereference(reference, (URL) basePath); + return dereference(reference, (URL) basePath, encoding); } else - return dereference(new File(reference), (Path)basePath, isArchivePackage); + return dereference(new File(reference), (Path)basePath, encoding, isArchivePackage); } return null; diff --git a/src/main/java/io/frictionlessdata/datapackage/Package.java b/src/main/java/io/frictionlessdata/datapackage/Package.java index 87c6105..59f1719 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Package.java +++ b/src/main/java/io/frictionlessdata/datapackage/Package.java @@ -145,8 +145,8 @@ public Package(URL urlSource, boolean strict) throws Exception { if (!isValidUrl(urlSource.toExternalForm())) { throw new DataPackageException("URL form not valid: "+urlSource.toExternalForm()); } - // Get string content of given remove file. - String jsonString = getFileContentAsString(urlSource); + // Get string content of given remote file. + String jsonString = getFileContentAsString(urlSource, StandardCharsets.UTF_8); // Create JsonNode and validate. try { @@ -193,10 +193,12 @@ public Package(Path descriptorFile, boolean strict) throws Exception { if (isArchive(descriptorFile.toFile())) { isArchivePackage = true; basePath = descriptorFile; - sourceJsonNode = createNode(JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME)); + sourceJsonNode = createNode( + JSONBase.getZipFileContentAsString(descriptorFile, DATAPACKAGE_FILENAME, StandardCharsets.UTF_8) + ); } else { basePath = descriptorFile.getParent(); - String sourceJsonString = getFileContentAsString(descriptorFile); + String sourceJsonString = getFileContentAsString(descriptorFile, StandardCharsets.UTF_8); sourceJsonNode = createNode(sourceJsonString); } this.setJson((ObjectNode) sourceJsonNode); diff --git a/src/main/java/io/frictionlessdata/datapackage/Validator.java b/src/main/java/io/frictionlessdata/datapackage/Validator.java index 5ba8bbd..4d24b0e 100644 --- a/src/main/java/io/frictionlessdata/datapackage/Validator.java +++ b/src/main/java/io/frictionlessdata/datapackage/Validator.java @@ -61,7 +61,7 @@ public static void validate(JsonNode jsonObjectToValidate, String profileId) thr schema.validate(jsonObjectToValidate); // throws a ValidationException if this object is invalid }else{ - throw new DataPackageException("Invalid profile id: " + profileId); + throw new ValidationException("Invalid profile id: " + profileId); } } @@ -81,7 +81,7 @@ public static void validate(JsonNode jsonObjectToValidate, URL schemaUrl) throws schema.validate(jsonObjectToValidate); // throws a ValidationException if this object is invalid }catch(FileNotFoundException e){ - throw new DataPackageException("Invalid profile schema URL: " + schemaUrl); + throw new ValidationException("Invalid profile schema URL: " + schemaUrl); } } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java index 3dfd418..8325043 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractDataResource.java @@ -4,7 +4,9 @@ import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.tableschema.Table; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; @@ -23,7 +25,7 @@ public abstract class AbstractDataResource extends AbstractResource { AbstractDataResource(String name, T data) { super(name); this.data = data; - super.format = Resource.FORMAT_JSON; + super.format = TableDataSource.Format.FORMAT_JSON.getLabel(); serializeToFile = false; if (data == null) throw new DataPackageException("Invalid Resource. The data property cannot be null for a Data-based Resource."); @@ -45,7 +47,7 @@ public void setDataPoperty(T data) { } @Override - List readData () throws Exception{ + List
readData () { List
tables = new ArrayList<>(); if (data != null){ if (format.equalsIgnoreCase(getResourceFormat())){ diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java index ff67d6b..312c3e0 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractReferencebasedResource.java @@ -65,7 +65,7 @@ public Set getDatafileNamesForWriting() { }).collect(Collectors.toSet()); } - abstract Table createTable(T reference) throws Exception; + abstract Table createTable(T reference, Charset encoding) throws Exception; abstract String getStringRepresentation(T reference); diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java index 8f4e0e9..3e90ee2 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/AbstractResource.java @@ -21,12 +21,14 @@ import io.frictionlessdata.tableschema.util.JsonUtil; import org.apache.commons.collections4.iterators.IteratorChain; import org.apache.commons.csv.CSVFormat; +import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.io.OutputStream; import java.io.Writer; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -480,6 +482,19 @@ public void setSchema(Schema schema) { this.schema = schema; } + + public Charset getEncodingOrDefault() { + String encoding = super.getEncoding(); + Charset cs; + if (!StringUtils.isEmpty(encoding)) { + cs = Charset.forName(encoding); + } else { + cs = StandardCharsets.UTF_8; + super.setEncoding(cs.name()); + } + return cs; + } + @JsonIgnore public String getDialectReference() { if (null == originalReferences.get(JSONBase.JSON_KEY_DIALECT)) diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java deleted file mode 100644 index 2ab82e8..0000000 --- a/src/main/java/io/frictionlessdata/datapackage/resource/CSVDataResource.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.frictionlessdata.datapackage.resource; - -public class CSVDataResource extends AbstractDataResource { - - public CSVDataResource(String name, String data) { - super(name, data); - super.format = getResourceFormat(); - } - - @Override - String getResourceFormat() { - return Resource.FORMAT_CSV; - } -} diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/CSVTabularDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/CSVTabularDataResource.java new file mode 100644 index 0000000..ca12ab1 --- /dev/null +++ b/src/main/java/io/frictionlessdata/datapackage/resource/CSVTabularDataResource.java @@ -0,0 +1,18 @@ +package io.frictionlessdata.datapackage.resource; + +import io.frictionlessdata.datapackage.Profile; +import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; + +public class CSVTabularDataResource extends AbstractDataResource { + + public CSVTabularDataResource(String name, String data) { + super(name, data); + super.format = getResourceFormat(); + super.profile = Profile.PROFILE_TABULAR_DATA_RESOURCE; + } + + @Override + String getResourceFormat() { + return TableDataSource.Format.FORMAT_CSV.getLabel(); + } +} diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java index 67bffb2..ccee1fb 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/FilebasedResource.java @@ -7,9 +7,11 @@ import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; +import org.apache.commons.lang3.StringUtils; import java.io.File; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; @@ -92,7 +94,7 @@ public File getBasePath() { } @Override - Table createTable(File reference) throws Exception { + Table createTable(File reference, Charset encoding) { return Table.fromSource(reference, basePath, schema, getCsvFormat()); } @@ -116,9 +118,10 @@ List
readData () throws Exception{ private List
readfromZipFile() throws Exception { List
tables = new ArrayList<>(); + Charset cs = getEncodingOrDefault(); for (File file : paths) { String fileName = file.getPath().replaceAll("\\\\", "/"); - String content = getZipFileContentAsString (basePath.toPath(), fileName); + String content = getZipFileContentAsString (basePath.toPath(), fileName, cs); Table table = Table.fromSource(content, schema, getCsvFormat()); tables.add(table); } @@ -135,38 +138,14 @@ private List
readfromOrdinaryFile() throws Exception { */ Path securePath = Resource.toSecure(file.toPath(), basePath.toPath()); Path relativePath = basePath.toPath().relativize(securePath); - Table table = createTable(relativePath.toFile()); + Charset cs = getEncodingOrDefault(); + + Table table = createTable(relativePath.toFile(), cs); tables.add(table); } return tables; } -/* - @Override - public void writeDataAsCsv(Path outputDir, Dialect dialect) throws Exception { - Dialect lDialect = (null != dialect) ? dialect : Dialect.DEFAULT; - List paths = new ArrayList<>(getReferencesAsStrings()); - int cnt = 0; - for (String fileName : paths) { - List
tables = getTables(); - Table t = tables.get(cnt++); - Path p; - if (outputDir.toString().isEmpty()) { - p = outputDir.getFileSystem().getPath(fileName); - if (!Files.exists(p)) { - Files.createDirectories(p); - } - } else { - if (!Files.exists(outputDir)) { - Files.createDirectories(outputDir); - } - p = outputDir.resolve(fileName); - } - Files.deleteIfExists(p); - writeTableAsCsv(t, lDialect, p); - } - } - */ public void setIsInArchive(boolean isInArchive) { this.isInArchive = isInArchive; } diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java index 7cc16da..a380a8d 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/JSONDataResource.java @@ -9,6 +9,7 @@ public class JSONDataResource extends AbstractDataResource { public JSONDataResource(String name, String json) { super(name, JsonUtil.getInstance().createArrayNode(json)); super.format = getResourceFormat(); + super.profile = "data-resource"; } @Override diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java index 7f0e54d..e060b5f 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/Resource.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import io.frictionlessdata.datapackage.Dialect; import io.frictionlessdata.datapackage.JSONBase; +import io.frictionlessdata.datapackage.Profile; import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.datapackage.exceptions.DataPackageValidationException; import io.frictionlessdata.tableschema.Table; @@ -15,7 +16,6 @@ import io.frictionlessdata.tableschema.tabledatasource.TableDataSource; import io.frictionlessdata.tableschema.util.JsonUtil; import org.apache.commons.lang3.StringUtils; -import org.locationtech.jts.io.OutStream; import java.io.*; import java.net.MalformedURLException; @@ -36,9 +36,6 @@ */ public interface Resource { - String FORMAT_CSV = "csv"; - String FORMAT_JSON = "json"; - /** * Return the {@link Table} objects underlying the Resource. * @return Table(s) @@ -395,12 +392,14 @@ static AbstractResource build(ObjectNode resourceJson, Object basePath, boolean Object data = resourceJson.get(JSONBase.JSON_KEY_DATA); String format = textValueOrNull(resourceJson, JSONBase.JSON_KEY_FORMAT); Dialect dialect = JSONBase.buildDialect (resourceJson, basePath, isArchivePackage); - Schema schema = JSONBase.buildSchema(resourceJson, basePath, isArchivePackage); String encoding = textValueOrNull(resourceJson, JSONBase.JSON_KEY_ENCODING); Charset charset = TableDataSource.getDefaultEncoding(); if (StringUtils.isNotEmpty(encoding)) { charset = Charset.forName(encoding); } + Schema schema = JSONBase.buildSchema(resourceJson, basePath, charset, isArchivePackage); + String profile = textValueOrNull(resourceJson, JSONBase.JSON_KEY_PROFILE); + // Now we can build the resource objects AbstractResource resource = null; @@ -408,6 +407,13 @@ static AbstractResource build(ObjectNode resourceJson, Object basePath, boolean if (path != null){ Collection paths = fromJSON(path, basePath); resource = build(name, paths, basePath, charset); + if ((null != resource.getSerializationFormat()) + && (resource.getSerializationFormat().equals(TableDataSource.Format.FORMAT_CSV.getLabel())) + ) { + resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); + } else { + resource.setProfile(profile); + } if (resource instanceof FilebasedResource) { ((FilebasedResource)resource).setIsInArchive(isArchivePackage); } @@ -422,10 +428,14 @@ static AbstractResource build(ObjectNode resourceJson, Object basePath, boolean "Invalid Resource. The format property cannot be null for inlined CSV data."); } resource = new JSONDataResource(name, data.toString()); - } else if (format.equals(Resource.FORMAT_JSON)) + resource.setProfile(profile); + } else if (format.equals(TableDataSource.Format.FORMAT_JSON.getLabel())) { resource = new JSONDataResource(name, data.toString()); - else if (format.equals(Resource.FORMAT_CSV)) - resource = new CSVDataResource(name, data.toString()); + resource.setProfile(profile); + }else if (format.equals(TableDataSource.Format.FORMAT_CSV.getLabel())) { + //CSV resources are always tabular. + resource = new CSVTabularDataResource(name, data.toString()); + } } else { throw new DataPackageValidationException( "Invalid Resource. The path property or the data and format properties cannot be null."); diff --git a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java index 1ba23fb..33ebe40 100644 --- a/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java +++ b/src/main/java/io/frictionlessdata/datapackage/resource/URLbasedResource.java @@ -16,8 +16,8 @@ public URLbasedResource(String name, Collection paths) { } @Override - Table createTable(URL reference) throws Exception { - return Table.fromSource(reference, schema, getCsvFormat()); + Table createTable(URL reference, Charset encoding) throws Exception { + return Table.fromSource(reference, schema, getCsvFormat(), encoding); } @Override @@ -28,10 +28,11 @@ String getStringRepresentation(URL reference) { @Override List
readData () throws Exception{ List
tables = new ArrayList<>(); + Charset cs = getEncodingOrDefault(); // If the path of a data file has been set. if (super.paths != null){ for (URL url : paths) { - Table table = createTable(url); + Table table = createTable(url, cs); tables.add(table); } } diff --git a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java index 00e54fb..e08ae89 100644 --- a/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/ValidatorTest.java @@ -4,62 +4,52 @@ import io.frictionlessdata.datapackage.exceptions.DataPackageException; import io.frictionlessdata.tableschema.exception.ValidationException; import io.frictionlessdata.tableschema.util.JsonUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import static org.junit.jupiter.api.Assertions.assertThrows; + /** * Test calls for JSON Validator class. * */ public class ValidatorTest { private static URL url; - - @Rule - public final ExpectedException exception = ExpectedException.none(); - - private Validator validator = null; - @Before - public void setup() throws MalformedURLException { - validator = new Validator(); + @BeforeAll + public static void setup() throws MalformedURLException { url = new URL("https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/test/resources/fixtures/datapackages/multi-data/datapackage.json"); } - + @Test public void testValidatingInvalidJsonObject() throws IOException, DataPackageException { JsonNode datapackageJsonObject = JsonUtil.getInstance().createNode("{\"invalid\" : \"json\"}"); - - exception.expect(ValidationException.class); - validator.validate(datapackageJsonObject); + + assertThrows(ValidationException.class, () -> {Validator.validate(datapackageJsonObject);}); } @Test public void testValidatingInvalidJsonString() throws IOException, DataPackageException{ String datapackageJsonString = "{\"invalid\" : \"json\"}"; - - exception.expect(ValidationException.class); - validator.validate(datapackageJsonString); + + assertThrows(ValidationException.class, () -> {Validator.validate(datapackageJsonString);}); } - + @Test public void testValidationWithInvalidProfileId() throws Exception { Package dp = new Package(url, true); String invalidProfileId = "INVALID_PROFILE_ID"; dp.setProperty("profile", invalidProfileId); - - exception.expectMessage("Invalid profile id: " + invalidProfileId); - dp.validate(); + Exception ex = assertThrows(ValidationException.class, () -> {dp.validate();}); + Assertions.assertEquals("Invalid profile id: " + invalidProfileId, ex.getMessage()); } - @Test public void testValidationWithValidProfileUrl() throws Exception { Package dp = new Package(url, true); @@ -69,7 +59,7 @@ public void testValidationWithValidProfileUrl() throws Exception { dp.validate(); // No exception thrown, test passes. - Assert.assertEquals("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/" + + Assertions.assertEquals("https://raw.githubusercontent.com/frictionlessdata/datapackage-java/" + "master/src/main/resources/schemas/data-package.json", dp.getProfile()); } @@ -80,8 +70,8 @@ public void testValidationWithInvalidProfileUrl() throws Exception { String invalidProfileUrl = "https://raw.githubusercontent.com/frictionlessdata/datapackage-java" + "/master/src/main/resources/schemas/INVALID.json"; dp.setProperty("profile", invalidProfileUrl); - - exception.expectMessage("Invalid profile schema URL: " + invalidProfileUrl); - dp.validate(); + + Exception ex = assertThrows(ValidationException.class, () -> {dp.validate();}); + Assertions.assertEquals("Invalid profile schema URL: " + invalidProfileUrl, ex.getMessage()); } } diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java index 37cdf3f..0930d0a 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/ResourceTest.java @@ -220,7 +220,7 @@ public void testIterateDataWithCast() throws Exception{ @Test public void testIterateDataFromCsvFormat() throws Exception{ String dataString = "city,year,population\nlondon,2017,8780000\nparis,2017,2240000\nrome,2017,2860000"; - Resource resource = new CSVDataResource("population", dataString); + Resource resource = new CSVTabularDataResource("population", dataString); // Set the profile to tabular data resource. resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE); diff --git a/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java index bb85b37..67e0fbd 100644 --- a/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java +++ b/src/test/java/io/frictionlessdata/datapackage/resource/RoundtripTest.java @@ -57,7 +57,7 @@ public void dogfoodingTest() throws Exception { new File(TestUtil.getBasePath().toFile(), "/schema/population_schema.json"), true); res.setSchema(schema); res.setShouldSerializeToFile(true); - res.setSerializationFormat(Resource.FORMAT_CSV); + res.setSerializationFormat(TableDataSource.Format.FORMAT_CSV.getLabel()); res.setDialect(Dialect.fromCsvFormat(csvFormat)); pkg.addResource(res);