Java Monitor S3 File Update Time

Java Monitor S3 File Update Time

I have a JAVA Spring Boot Project. The requirement is to monitor the file update time in S3.

Here is the pom.xml requirement pom.xml
<!-- AWS -->
<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk-s3</artifactId>
    <version>1.11.251</version>
</dependency>

Here is the implementation Class S3ServiceImpl.java

package com.sillycat.jobsmonitorapi.service;

import java.util.Date;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.sillycat.jobsmonitorapi.config.AWSConfig;

@Service
public class S3ServiceImpl implements S3Service {

    private static final Logger logger = LoggerFactory.getLogger(S3ServiceImpl.class);

    @Autowired
    private AWSConfig awsConfig;

    private AmazonS3 s3Client;

    private void init() {
        logger.info("start to init the s3Client--------");
        this.s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(
                        new BasicAWSCredentials(awsConfig.getKey(), this.awsConfig.getSecret())))
                .withRegion(Regions.US_EAST_1).build();
        logger.info("start to init the s3Client end--------");
    }

    public Date getFileLastUpdate(String bucketName, String path) {
        if (this.s3Client == null) {
            this.init();
        }
        Date result = null;
        ObjectListing objectList = this.s3Client.listObjects(bucketName, path);
        if (objectList != null) {
            for (S3ObjectSummary fileSummary : objectList.getObjectSummaries()) {
                String tmpKey = fileSummary.getKey();
                logger.info("find files {}", tmpKey);
                logger.info("file the file with last update {}", fileSummary.getLastModified());
                result = fileSummary.getLastModified();
                break;
            }
        }
        return result;
    }

}

Here is the config class under the config directory AWSConfig.java
package com.sillycat.jobsmonitorapi.config;

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;

@Configuration
@EnableConfigurationProperties
@ConfigurationProperties(prefix = "aws")
public class AWSConfig {

    private String key;

    private String secret;

    public String getKey() {
        return key;
    }

    public void setKey(String key) {
        this.key = key;
    }

    public String getSecret() {
        return secret;
    }

    public void setSecret(String secret) {
        this.secret = secret;
    }

}

In side the application.yaml, we will have our configuration as follow:
aws:
  key: xxxxxxx
  secret: xxxxxxxxxxxx

Here is the UNIT test how it will pick up the configuration and work S3ServiceTest.java

package com.sillycat.jobsmonitorapi.service;

import java.util.Date;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;

@RunWith(SpringRunner.class)
@SpringBootTest
@ActiveProfiles("dev")
public class S3ServiceTest {

    @Autowired
    S3Service s3Service;

    @Test
    public void testDummy() {
        Assert.assertTrue(true);
    }

    @Test
    public void testLastModified() {
        Date date = s3Service.getFileLastUpdate("j2c-production", "feed-2g/jobs_2g.txt.gz");
        Assert.assertNotNull(date);
    }

}

Here is the Util Class to deal with the time, I tried to use JAVA API 8 this time, no JODA Time
package com.sillycat.jobsmonitorapi.commons;

import java.time.LocalDateTime;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

public class DateTimeUtil {

    private static final DateTimeFormatter formatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss");

    /**
     *
     * @param timeString
     * @return
     */
    public static DateTime parseFullTimeString(String timeString) {
        DateTime dt = formatter.parseDateTime(timeString);
        return dt;
    }

    /**
     *
     * @param date2
     *            time from S3
     * @param hours
     * @return
     */
    public static Boolean isUpdateDeleyHours(LocalDateTime updateTime, int hours) {
        if (updateTime.isBefore(LocalDateTime.now().minusHours(hours))) {
            // S3 is older than now - hours, something wrong
            return true;
        } else {
            return false;
        }
    }

}

Here is the unit class to handle the DateTimeUtil.java

package com.sillycat.jobsmonitorapi.commons;

import java.time.LocalDateTime;
import java.time.Month;
import java.time.ZoneId;
import java.util.Date;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Test;

public class DateTimeUtilTest {

    @Test
    public void dummy() {
        Assert.assertTrue(true);
    }

    @Test
    public void parseFullTimeString() {
        String timeString = "2017-11-02 10:55:37";
        DateTime dt = DateTimeUtil.parseFullTimeString(timeString);
        Assert.assertNotNull(dt);
    }

    @Test
    public void isDeleyHours() {
        LocalDateTime oldTime = LocalDateTime.of(2017, Month.JANUARY, 14, 19, 30);
        Boolean result1 = DateTimeUtil.isUpdateDeleyHours(oldTime, 3);
        Assert.assertTrue(result1);
        Date now = new Date();
        LocalDateTime nowDateTime = LocalDateTime.ofInstant(now.toInstant(), ZoneId.systemDefault());
        Boolean result2 = DateTimeUtil.isUpdateDeleyHours(nowDateTime, 3);
        Assert.assertFalse(result2);
       
    }

}



References:
http://docs.aws.amazon.com/AmazonS3/latest/dev/RetrievingObjectUsingJava.html
http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/S3ObjectSummary.html
https://stackoverflow.com/questions/38578937/spring-boot-amazon-aws-s3-bucket-file-download-access-denied