Обнаружена ошибка при выполнении шага MasterStep Execption Загрузка данных в MySQL Spring Batch - PullRequest
Новая
       37

Обнаружена ошибка при выполнении шага MasterStep Execption Загрузка данных в MySQL Spring Batch

0 голосов
/ 26 марта 2019

Я пытаюсь загрузить набор файлов CSV в MySQL с помощью Spring Batch.Мой код приведен ниже.Но когда я запускаю 2 или 3 файла, данные загружаются нормально.но когда я пытаюсь с 100 или более его выкидывать исключение после загрузки 4 до 8 файлов данных.Исключение: 

ERROR 11633 --- [           main] o.s.batch.core.step.AbstractStep         : Encountered an error executing step masterStep in job importUserJob
org.springframework.batch.core.JobExecutionException: Partition handler returned an unsuccessful step
        at org.springframework.batch.core.partition.support.PartitionStep.doExecute(PartitionStep.java:112) ~[spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.step.AbstractStep.execute(AbstractStep.java:200) ~[spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.SimpleStepHandler.handleStep(SimpleStepHandler.java:148) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.flow.JobFlowExecutor.executeStep(JobFlowExecutor.java:66) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.flow.support.state.StepState.handle(StepState.java:67) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.flow.support.SimpleFlow.resume(SimpleFlow.java:169) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.flow.support.SimpleFlow.start(SimpleFlow.java:144) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.flow.FlowJob.doExecute(FlowJob.java:136) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.job.AbstractJob.execute(AbstractJob.java:308) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.batch.core.launch.support.SimpleJobLauncher$1.run(SimpleJobLauncher.java:141) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.core.task.SyncTaskExecutor.execute(SyncTaskExecutor.java:50) [spring-core-5.0.5.RELEASE.jar!/:5.0.5.RELEASE]
        at org.springframework.batch.core.launch.support.SimpleJobLauncher.run(SimpleJobLauncher.java:134) [spring-batch-core-4.0.1.RELEASE.jar!/:4.0.1.RELEASE]
        at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.execute(JobLauncherCommandLineRunner.java:163) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.executeLocalJobs(JobLauncherCommandLineRunner.java:179) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.launchJobFromProperties(JobLauncherCommandLineRunner.java:134) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.autoconfigure.batch.JobLauncherCommandLineRunner.run(JobLauncherCommandLineRunner.java:128) [spring-boot-autoconfigure-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:797) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:781) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.SpringApplication.run(SpringApplication.java:335) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]
        at org.springframework.boot.builder.SpringApplicationBuilder.run(SpringApplicationBuilder.java:137) [spring-boot-2.0.1.RELEASE.jar!/:2.0.1.RELEASE]

Я не знаю, что происходит позади, Любая помощь приветствуется.Код: 

@Configuration
@EnableBatchProcessing
public class ImportJobConfig {

    @Autowired
    private JobBuilderFactory jobBuilderFactory;
    @Autowired
    private StepBuilderFactory stepBuilderFactory;
    @Autowired
    private DataSource dataSource;
    @Autowired
    private JdbcBatchItemWriter<Person> writer;
    @Autowired
    private FlatFileItemReader<Person> personItemReader;
    @Bean
    public DataSource dataSource() {
        final DriverManagerDataSource dataSource = new DriverManagerDataSource();
        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
        dataSource.setUrl("URL");
        dataSource.setUsername("USERNAME");
        dataSource.setPassword("PASSWORD");
        return dataSource;
    }
    @Bean
    public ThreadPoolTaskExecutor taskExecutor() {
        ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
        taskExecutor.setMaxPoolSize(4);
        taskExecutor.setCorePoolSize(4);
        taskExecutor.setQueueCapacity(4);
        taskExecutor.afterPropertiesSet();
        taskExecutor.setAllowCoreThreadTimeOut(true);
        return taskExecutor;
    }
    @Bean
    @Qualifier("masterStep")
    public Step masterStep() {
        return stepBuilderFactory.get("masterStep").partitioner("step1", partitioner()).step(step1())
                .taskExecutor(taskExecutor()).build();
    }
    @Bean("partitioner")
    @StepScope
    public Partitioner partitioner() {
        System.out.println("In Partitioner");
        MultiResourcePartitioner partitioner = new MultiResourcePartitioner();
        ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
        String filePath = "//data//person//*.csv";
        Resource[] resources = null;
        try {
            resources = resolver.getResources("file:" + filePath);
        } catch (IOException e) {
            e.printStackTrace();
        }
        partitioner.setResources(resources);
        partitioner.partition(4);
        return partitioner;
    }
    @Bean
    public APSUploadFileItemProcessor processor() {
        return new APSUploadFileItemProcessor();
    }
    @Bean
    public JdbcBatchItemWriter<Person> writer() {
        JdbcBatchItemWriter<Person> writer = new JdbcBatchItemWriter<>();
        try {
            writer.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<Person>());
            writer.setSql("INSERT INTO person(name,business,phone) VALUES(name, :business, :phone)");
            writer.setDataSource(dataSource);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return writer;
    }
    @Bean
    public Job importUserJob(JobCompletionNotificationListener listener, Step step1) {
        return jobBuilderFactory.get("importUserJob").incrementer(new RunIdIncrementer()).listener(listener)
                .flow(masterStep()).end().build();
    }
    @Bean
    public Step step1() {
        return stepBuilderFactory.get("step1").<Person, Person>chunk(10000).processor(processor())
                .writer(writer).reader(personItemReader).build();
    }
    @Bean
    @StepScope
    @Qualifier("personItemReader")
    @DependsOn("partitioner")
    public FlatFileItemReader<Person> personItemReader(
            @Value("#{stepExecutionContext['fileName']}") String filename) throws MalformedURLException {
        return new FlatFileItemReaderBuilder<Person>().name("personItemReader").delimited().delimiter("|")
                .names(new String[] { "name","business","phone" })
                .fieldSetMapper(new BeanWrapperFieldSetMapper<Person>() {
                    {
                        setTargetType(Person.class);
                    }
                }).resource(new UrlResource(filename)).build();
    }
}
...