Java Generate/Merge Files(1)Redis Content Fetch

Java Generate/Merge Files(1)Redis Content Fetch

Fetch Data from Redis
https://github.com/xetorthio/jedis
https://github.com/lettuce-io/lettuce-core

I first use lettuce, it is not that well document as JEDIS and I tried with uncompress the data I gzcompress in PHP. I can not uncompress that. So I finally use JEDIS instead.

Some Lettuce-core codes for future references.
package com.j2c.feeds2g.services;

import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import com.lambdaworks.redis.RedisFuture;
import com.lambdaworks.redis.RedisURI;
import com.lambdaworks.redis.ValueScanCursor;
import com.lambdaworks.redis.api.async.RedisSetAsyncCommands;
import com.lambdaworks.redis.api.async.RedisStringAsyncCommands;
import com.lambdaworks.redis.cluster.RedisClusterClient;
import com.lambdaworks.redis.cluster.api.StatefulRedisClusterConnection;

public class RedisLettuceServiceImpl implements RedisService{


RedisClusterClient redisClusterClient;

public void processJobsBySource(Integer sourceID) {
StatefulRedisClusterConnection<String, String> connection = redisClusterClient.connect();

RedisStringAsyncCommands<String, String> async = connection.async();





RedisFuture<String> smembers = async.getrange("source_referencenumbers_1299", 0, 10);

try {

smembers.get(10, TimeUnit.SECONDS);


} catch (InterruptedException e) {

e.printStackTrace();


} catch (ExecutionException e) {

e.printStackTrace();


} catch (TimeoutException e) {

e.printStackTrace();


}

}

public void processJobsByBucket(String bucketJobIDs) {




}

public void setRedisClusterClient(RedisClusterClient redisClusterClient) {
this.redisClusterClient = redisClusterClient;

}






public static void main(String[] args) {
RedisURI redisUri = RedisURI.Builder.redis("stage-jobs-c.cache.amazonaws.com").build();




RedisClusterClient clusterClient = RedisClusterClient.create(redisUri);
StatefulRedisClusterConnection<String, String> connection = clusterClient.connect();

RedisSetAsyncCommands<String, String> setAsync = connection.async();

RedisFuture<ValueScanCursor<String>> results = setAsync.sscan("source_referencenumbers_1299");

RedisStringAsyncCommands<String, String> async = connection.async();

try {

ValueScanCursor<String> contents = results.get(10, TimeUnit.SECONDS);


List<String> jobIDs = contents.getValues();


System.out.println(jobIDs);








String jobID = "jobinfo_1299_" + jobIDs.get(0);


RedisFuture<String> results2 = async.get(jobID);








System.out.println("key  = " + jobID);








String jobInfo = results2.get(10, TimeUnit.SECONDS);








System.out.println("raw job info = " + jobInfo);


} catch (InterruptedException e) {

e.printStackTrace();


} catch (ExecutionException e) {

e.printStackTrace();


} catch (TimeoutException e) {

e.printStackTrace();


}

}


}

            <dependency>
                    <groupId>biz.paluch.redis</groupId>
                    <artifactId>lettuce</artifactId>
                    <version>4.3.1.Final</version>
            </dependency>

Try to format the unit tests the JEDIS codes. For JEDIS, I am using  JedisCluster.
And I want to use  JedisPool as well, but it seems that JedisPool does not support JedisCluster, I am implement one myself with object common pool.

Some JedisPool codes

package com.j2c.feeds2g.services;

import java.util.HashSet;
import java.util.Set;

import com.j2c.feeds2g.commons.utils.CompressUtil;

import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisCluster;

public class RedisJRedisServiceImpl implements RedisService {

    public void processJobsBySource(Integer sourceID) {

    }

    public void processJobsByBucket(String bucketJobIDs) {

    }

    public static void main(String[] args) {

        Set<HostAndPort> jedisClusterNodes = new HashSet<HostAndPort>();
        // Jedis Cluster will attempt to discover cluster nodes automatically
        jedisClusterNodes.add(new HostAndPort("stage-jobs-c.pnuura.clustercfg.use1.cache.amazonaws.com", 6379));
        JedisCluster jedis = new JedisCluster(jedisClusterNodes);

        // JedisPoolConfig poolConfig = new JedisPoolConfig();
        // poolConfig.setMaxTotal(1000);
        // poolConfig.setMaxIdle(10);
        // poolConfig.setMinIdle(1);
        // poolConfig.setMaxWaitMillis(30000);
        // JedisPool jedisPool = new JedisPool(poolConfig,
        // "stage-jobs-c.pnuura.com",
        // 6379,
        // 1000
        // );
        //
        // Jedis jedis = jedisPool.getResource();

        // ScanParams scanParams = new ScanParams().count(100);
        // String cursor = "0";
        // int count = 0;
        // do {
        // count = count + 100;
        // ScanResult<String> scanResult =
        // jredisCluster.sscan("source_referencenumbers_1299", cursor,
        // scanParams);
        // cursor = scanResult.getStringCursor();
        // List<String> jobIDs = scanResult.getResult();
        // if (!jobIDs.isEmpty()) {
        // System.out.println(count + " " + jobIDs.get(0));
        // }
        // } while (!"0".equals(cursor));

        String jobID = "jobinfo_1299_1679_5e36e4a78e5cc158cd48de067ef085e0";
        byte[] jobinfo = jedis.get(jobID.getBytes());
        byte[] jobinfo2 = CompressUtil.gzuncompress(jobinfo);

        String jobinfo3 = new String(jobinfo2);
        System.out.println(jobinfo3);
    }

}

References:
http://www.mkyong.com/spring-batch/spring-batch-and-spring-taskscheduler-example/

compress java
http://type.so/java/php-gzuncompress-in-java.html

spring with JEDIS
http://docs.spring.io/spring-data/redis/docs/1.8.1.RELEASE/reference/html/

http://projects.spring.io/spring-data-redis/#quick-start

logging set up
https://wiki.base22.com/display/btg/How+to+setup+SLF4J+and+LOGBack+in+a+web+app+-+fast