From 9a38b6edc34c1f283bfafeed4c5324ce4deb809b Mon Sep 17 00:00:00 2001 From: Alexey Polyakov Date: Mon, 12 Feb 2024 20:06:39 +0300 Subject: [PATCH 1/2] Realize upload and save cities --- config_templates/sql/initDB.sql | 12 +++- .../masterjava/persist/dao/CityDao.java | 42 +++++++++++++ .../masterjava/persist/dao/ProjectDao.java | 5 ++ .../masterjava/persist/dao/UserDao.java | 2 +- .../masterjava/persist/model/City.java | 16 +++++ .../masterjava/persist/model/Group.java | 19 ++++++ .../masterjava/persist/model/Project.java | 17 ++++++ .../masterjava/persist/model/TypeGroup.java | 6 ++ persist/src/main/resources/persist.conf | 2 +- .../masterjava/persist/CityTestData.java | 42 +++++++++++++ .../masterjava/persist/dao/CityDaoTest.java | 45 ++++++++++++++ .../masterjava/upload/CityProcessor.java | 59 +++++++++++++++++++ .../masterjava/upload/UploadServlet.java | 3 +- 13 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/dao/CityDao.java create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/model/City.java create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/model/Group.java create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/model/Project.java create mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/model/TypeGroup.java create mode 100644 persist/src/test/java/ru/javaops/masterjava/persist/CityTestData.java create mode 100644 persist/src/test/java/ru/javaops/masterjava/persist/dao/CityDaoTest.java create mode 100644 web/upload/src/main/java/ru/javaops/masterjava/upload/CityProcessor.java diff --git a/config_templates/sql/initDB.sql b/config_templates/sql/initDB.sql index 09b463d..19c08fe 100644 --- a/config_templates/sql/initDB.sql +++ b/config_templates/sql/initDB.sql @@ -1,10 +1,12 @@ DROP TABLE IF EXISTS users; DROP SEQUENCE IF EXISTS user_seq; DROP TYPE IF EXISTS user_flag; - CREATE TYPE user_flag AS ENUM ('active', 'deleted', 'superuser'); +DROP TABLE IF EXISTS cities; +DROP SEQUENCE IF EXISTS cities_seq; CREATE SEQUENCE user_seq START 100000; +CREATE SEQUENCE cities_seq START 100000; CREATE TABLE users ( id INTEGER PRIMARY KEY DEFAULT nextval('user_seq'), @@ -13,4 +15,10 @@ CREATE TABLE users ( flag user_flag NOT NULL ); -CREATE UNIQUE INDEX email_idx ON users (email); \ No newline at end of file +CREATE UNIQUE INDEX email_idx ON users (email); + +CREATE TABLE cities ( + id INTEGER PRIMARY KEY DEFAULT nextval('cities_seq'), + city TEXT NOT NULL +); +CREATE UNIQUE INDEX city_idx ON cities (city); \ No newline at end of file diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/dao/CityDao.java b/persist/src/main/java/ru/javaops/masterjava/persist/dao/CityDao.java new file mode 100644 index 0000000..4601376 --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/dao/CityDao.java @@ -0,0 +1,42 @@ +package ru.javaops.masterjava.persist.dao; + +import com.bertoncelj.jdbi.entitymapper.EntityMapperFactory; +import org.skife.jdbi.v2.sqlobject.*; +import org.skife.jdbi.v2.sqlobject.customizers.BatchChunkSize; +import org.skife.jdbi.v2.sqlobject.customizers.RegisterMapperFactory; +import ru.javaops.masterjava.persist.DBIProvider; +import ru.javaops.masterjava.persist.model.City; + +import java.util.List; + +@RegisterMapperFactory(EntityMapperFactory.class) +public abstract class CityDao implements AbstractDao { + + public City insert(City city) { + if (city.isNew()) { + int id = insertGeneratedId(city); + city.setId(id); + } else { + insertWithId(city); + } + return city; + } + @SqlQuery("SELECT nextval('cities_seq')") + public abstract int getNextVal(); + + @SqlUpdate("TRUNCATE cities") + @Override + public abstract void clean(); + + @SqlUpdate("INSERT INTO cities (city) VALUES (:city) ON CONFLICT DO NOTHING") + @GetGeneratedKeys + abstract int insertGeneratedId(@BindBean City city); + + @SqlUpdate("INSERT INTO cities (id, city) VALUES (:id, :city) ON CONFLICT DO NOTHING") + @GetGeneratedKeys + abstract int insertWithId(@BindBean City city); + + @SqlQuery("SELECT * FROM cities LIMIT :it") + abstract List getWithLimit(@Bind int limit); + +} diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java b/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java new file mode 100644 index 0000000..cfb1dde --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java @@ -0,0 +1,5 @@ +package ru.javaops.masterjava.persist.dao; + +public class ProjectDao { + +} diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/dao/UserDao.java b/persist/src/main/java/ru/javaops/masterjava/persist/dao/UserDao.java index 8932950..9bb1af0 100644 --- a/persist/src/main/java/ru/javaops/masterjava/persist/dao/UserDao.java +++ b/persist/src/main/java/ru/javaops/masterjava/persist/dao/UserDao.java @@ -26,7 +26,7 @@ public User insert(User user) { @SqlQuery("SELECT nextval('user_seq')") abstract int getNextVal(); - @Transaction + //@Transaction public int getSeqAndSkip(int step) { int id = getNextVal(); DBIProvider.getDBI().useHandle(h -> h.execute("ALTER SEQUENCE user_seq RESTART WITH " + (id + step))); diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/model/City.java b/persist/src/main/java/ru/javaops/masterjava/persist/model/City.java new file mode 100644 index 0000000..6204942 --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/model/City.java @@ -0,0 +1,16 @@ +package ru.javaops.masterjava.persist.model; + +import lombok.*; + +@Data +@EqualsAndHashCode(callSuper = true) +@RequiredArgsConstructor +@NoArgsConstructor +public class City extends BaseEntity { + private @NonNull String city; + + public City(Integer id, @NonNull String city) { + this(city); + this.id = id; + } +} diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/model/Group.java b/persist/src/main/java/ru/javaops/masterjava/persist/model/Group.java new file mode 100644 index 0000000..8c63e88 --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/model/Group.java @@ -0,0 +1,19 @@ +package ru.javaops.masterjava.persist.model; + +import com.bertoncelj.jdbi.entitymapper.Column; +import lombok.*; + +@Data +@RequiredArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class Group extends BaseEntity { + private @NonNull String name; + @Column("type_group") + private @NonNull TypeGroup typeGroup; + + public Group(Integer id, @NonNull String name, @NonNull TypeGroup typeGroup) { + this(name, typeGroup); + this.id = id; + } +} diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/model/Project.java b/persist/src/main/java/ru/javaops/masterjava/persist/model/Project.java new file mode 100644 index 0000000..0b46a80 --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/model/Project.java @@ -0,0 +1,17 @@ +package ru.javaops.masterjava.persist.model; + +import lombok.*; + +@Data +@RequiredArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class Project extends BaseEntity { + private @NonNull String name; + private @NonNull String description; + + public Project(Integer id, @NonNull String name, @NonNull String description) { + this(name, description); + this.id = id; + } +} diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/model/TypeGroup.java b/persist/src/main/java/ru/javaops/masterjava/persist/model/TypeGroup.java new file mode 100644 index 0000000..ede17c9 --- /dev/null +++ b/persist/src/main/java/ru/javaops/masterjava/persist/model/TypeGroup.java @@ -0,0 +1,6 @@ +package ru.javaops.masterjava.persist.model; + +public enum TypeGroup { + FINISHED, + CURRENT +} diff --git a/persist/src/main/resources/persist.conf b/persist/src/main/resources/persist.conf index fe15029..a396e3d 100644 --- a/persist/src/main/resources/persist.conf +++ b/persist/src/main/resources/persist.conf @@ -4,4 +4,4 @@ db { password = password } -include required(file("/apps/masterjava/config/persist.conf")) +#include required(file("/apps/masterjava/config/persist.conf")) diff --git a/persist/src/test/java/ru/javaops/masterjava/persist/CityTestData.java b/persist/src/test/java/ru/javaops/masterjava/persist/CityTestData.java new file mode 100644 index 0000000..79f79b7 --- /dev/null +++ b/persist/src/test/java/ru/javaops/masterjava/persist/CityTestData.java @@ -0,0 +1,42 @@ +package ru.javaops.masterjava.persist; + +import com.google.common.collect.ImmutableList; +import ru.javaops.masterjava.persist.DBIProvider; +import ru.javaops.masterjava.persist.dao.CityDao; +import ru.javaops.masterjava.persist.model.City; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.of; + +public class CityTestData { + public static City SPB; + public static City MSK; + public static City KIV; + public static City MNSK; + public static City Vladimir; + public static List fourCityList; + + public static void init() { + SPB = new City("Санкт-Петербург"); + MSK = new City("Москва"); + KIV = new City("Киев"); + MNSK = new City("Минск"); + Vladimir = new City("Владимир"); + fourCityList = ImmutableList.of(SPB, MSK, KIV, MNSK); + } + + public static void setUp() { + CityDao dao = DBIProvider.getDao(CityDao.class); + dao.clean(); + DBIProvider.getDBI().useTransaction((conn, status) -> { + fourCityList.forEach(dao::insert); + dao.insert(Vladimir); + }); + } + + public static City getNew() { + return new City("Екатеринбург"); + } + +} diff --git a/persist/src/test/java/ru/javaops/masterjava/persist/dao/CityDaoTest.java b/persist/src/test/java/ru/javaops/masterjava/persist/dao/CityDaoTest.java new file mode 100644 index 0000000..4ae1707 --- /dev/null +++ b/persist/src/test/java/ru/javaops/masterjava/persist/dao/CityDaoTest.java @@ -0,0 +1,45 @@ +package ru.javaops.masterjava.persist.dao; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import ru.javaops.masterjava.persist.CityTestData; +import ru.javaops.masterjava.persist.model.City; + +import java.util.List; + +import static ru.javaops.masterjava.persist.CityTestData.getNew; + +public class CityDaoTest extends AbstractDaoTest { + + + public CityDaoTest() { + super(CityDao.class); + } + + @BeforeClass + public static void beforeClass() throws Exception { + CityTestData.init(); + } + + @Before + public void setUp() throws Exception { + CityTestData.setUp(); + } + + @Test + public void testInsert() { + City expected = getNew(); + City actual = dao.insert(expected); + Assert.assertNotNull(actual); + expected.setId(actual.getId()); + Assert.assertSame(actual, expected); + } + + @Test + public void testGetWithLimit() { + List actual = dao.getWithLimit(4); + Assert.assertEquals(CityTestData.fourCityList, actual); + } +} \ No newline at end of file diff --git a/web/upload/src/main/java/ru/javaops/masterjava/upload/CityProcessor.java b/web/upload/src/main/java/ru/javaops/masterjava/upload/CityProcessor.java new file mode 100644 index 0000000..d49058c --- /dev/null +++ b/web/upload/src/main/java/ru/javaops/masterjava/upload/CityProcessor.java @@ -0,0 +1,59 @@ +package ru.javaops.masterjava.upload; + +import lombok.extern.slf4j.Slf4j; +import ru.javaops.masterjava.persist.DBIProvider; +import ru.javaops.masterjava.persist.dao.CityDao; +import ru.javaops.masterjava.persist.model.City; +import ru.javaops.masterjava.xml.schema.CityType; +import ru.javaops.masterjava.xml.schema.ObjectFactory; +import ru.javaops.masterjava.xml.util.JaxbParser; +import ru.javaops.masterjava.xml.util.JaxbUnmarshaller; +import ru.javaops.masterjava.xml.util.StaxStreamProcessor; + +import javax.xml.bind.JAXBException; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.events.XMLEvent; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +@Slf4j +public class CityProcessor { + private static final int NUMBER_THREAD = 4; + private final ExecutorService executorService = Executors.newFixedThreadPool(NUMBER_THREAD); + private static final JaxbParser JAXB_PARSER = new JaxbParser(ObjectFactory.class); + private static final CityDao cityDao = DBIProvider.getDao(CityDao.class); + + public void process(final InputStream is) throws XMLStreamException, JAXBException, IOException { + log.info("Start processing parse cities"); + is.reset(); + StaxStreamProcessor staxStreamProcessor = new StaxStreamProcessor(is); + JaxbUnmarshaller unmarshaller = JAXB_PARSER.createUnmarshaller(); + List> futuresCity = new ArrayList<>(); + while (staxStreamProcessor.doUntil(XMLEvent.START_ELEMENT, "City")) { + CityType cityType = unmarshaller.unmarshal(staxStreamProcessor.getReader(), CityType.class); + City city = new City(cityType.getValue()); + addCity(city, futuresCity); + } + List cities = new ArrayList<>(); + try { + for (Future f : futuresCity) { + cities.add(f.get()); + } + } catch (InterruptedException | ExecutionException e) { + + } + } + + private void addCity(City city, List> futuresCity) { + Future futureCity = executorService.submit(() -> cityDao.insert(city)); + futuresCity.add(futureCity); + log.info("city: {} insert", city.getCity()); + } + +} diff --git a/web/upload/src/main/java/ru/javaops/masterjava/upload/UploadServlet.java b/web/upload/src/main/java/ru/javaops/masterjava/upload/UploadServlet.java index 22c5f49..4c71840 100644 --- a/web/upload/src/main/java/ru/javaops/masterjava/upload/UploadServlet.java +++ b/web/upload/src/main/java/ru/javaops/masterjava/upload/UploadServlet.java @@ -23,7 +23,7 @@ public class UploadServlet extends HttpServlet { private static final int CHUNK_SIZE = 2000; private final UserProcessor userProcessor = new UserProcessor(); - + private final CityProcessor cityProcessor = new CityProcessor(); @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { out(req, resp, "", CHUNK_SIZE); @@ -42,6 +42,7 @@ protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws I Part filePart = req.getPart("fileToUpload"); try (InputStream is = filePart.getInputStream()) { List failed = userProcessor.process(is, chunkSize); + cityProcessor.process(is); log.info("Failed users: " + failed); final WebContext webContext = new WebContext(req, resp, req.getServletContext(), req.getLocale(), From 477e69229e46429f1c2a3cb3764d86df3173b63b Mon Sep 17 00:00:00 2001 From: Alexey Polyakov Date: Tue, 13 Feb 2024 19:33:44 +0300 Subject: [PATCH 2/2] Realize upload and save cities --- .../java/ru/javaops/masterjava/persist/dao/ProjectDao.java | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java diff --git a/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java b/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java deleted file mode 100644 index cfb1dde..0000000 --- a/persist/src/main/java/ru/javaops/masterjava/persist/dao/ProjectDao.java +++ /dev/null @@ -1,5 +0,0 @@ -package ru.javaops.masterjava.persist.dao; - -public class ProjectDao { - -}