大规模分布式系统架构与设计实战笔记8

用fourinone实现的基于整型读写的上亿排序

import com.fourinone.BeanContext;

public class ParkServerDemo
{
	public static void main(String[] args)
	{
		BeanContext.startPark();
	}
}

import com.fourinone.*;
import java.util.*;
import com.fourinone.ArrayAdapter.*;
//public class Worker extends MigrantWorker
public class Worker extends MigrantWorker
{
	int total=25000000;//总共的数据数
	int max=100000;//最大数据
	int block=100000;//每次处理多少数据
	int groups=16;//一共将数据分成多少组
	int groupnum=max/groups;//每组多少数据
	String path="d://tmp";//文件位置
	Workman[] wms=null;

	public Worker(){}
	public Worker(int total,int max,int block,int groups,String path)
	{
		this.total=total;
		this.max=max;
		this.block=block;
		this.groups=groups;
		this.groupnum=max/groups;
		this.path=path;
	}
	public WareHouse doTask(WareHouse wh)
	{

		int index=getSelfIndex();
		
		int step=(Integer)wh.getObj("step");

		if(wms==null)
		{
			wms=getWorkerAll();
		}
	    WareHouse result=new WareHouse("ok",1);
        //long begin=(new Date()).getTime();
		if(step==1)
		{
			//生成16个文件
			FileAdapter[] fas=new FileAdapter[groups];
			for(int i=0;i<groups;i++)
				fas[i]=new FileAdapter(path+"//"+index+"//"+i+"//data");
			//要处理的文件
			FileAdapter fa=new FileAdapter(path+"//"+index+"//data");
			for(int n=0;n<(total/block);n++)
			{
				//每次处理1000个数据
				int [] its=fa.getIntReader(n*block,block).readIntAll();
               //开辟一个ArrayList数组
				ArrayList<ArrayList<Integer>> list=new ArrayList<ArrayList<Integer>> ();
				for(int p=0;p<groups;p++)
				{
					list.add(new ArrayList<Integer>());
				}

				int temp=-1;
				for(int m=0;m<block;m++)
				{
					temp=its[m]/groupnum;
					//System.out.println(temp);
					list.get(temp).add(its[m]);	
				}
				for(int k=0;k<groups;k++)
				{
					fas[k].getIntWriter().writeListInt(list.get(k));
				}
			}
		}
		else if(step==2)
		{
			for(int i=0;i<16;i++)
			{
				FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
				int[] nums=fa.getIntReader().readIntAll();
				if((i/4)!=index)
				{
					WareHouse out=new WareHouse();
					out.put("i",i);
					out.put("v",nums);
					//将文件挪到(i/4)号节点的i文件中去
					wms[i/4].receive(out);		
				}
				fa.close();	
			}
		}
		else if(step==3)
		{
			int total=0;
			for(int i=0;i<16;i++)
			{
				
				if((i/4)==index)
				{
					FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
					int [] nums=fa.getIntReader().readIntAll();
					ListInt is=ArrayAdapter.getListInt();
					is.sort(nums);

					total+=nums.length;
                    

					FileAdapter tofa=new FileAdapter(path+"//"+index+"//"+i+".data");
					tofa.getIntWriter().writeInt(nums);
					System.out.println("写数据到"+i+".data");
					fa.close();
				}
				else
				{
					FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i);
					fa.delete();
					fa.close();
				}
				
			}
			result.setObj("total",total); 
		}
		return result;
	}
	protected boolean receive(WareHouse inhouse)
	{
		Integer i=(Integer)inhouse.get("i");
	    int [] nums=(int [])inhouse.get("v");
        int index=i/4;
        FileAdapter fa=new FileAdapter(path+"//"+index+"//"+i+"//data");
		fa.getIntWriter().writeInt(nums);
		System.out.println("文件"+i+"放到"+index+"节点上");
		return true;
	}

	public static void main(String [] args)
	{
		Worker w=new Worker(Integer.parseInt(args[2]),Integer.parseInt(args[3]),Integer.parseInt(args[4]),Integer.parseInt(args[5]),args[6]);
		w.waitWorking(args[0],Integer.parseInt(args[1]),"Worker");
	}

}

import com.fourinone.Contractor;
import com.fourinone.WareHouse;
import com.fourinone.WorkerLocal;
import java.util.Date;

public class Ctor extends Contractor
{
	public WareHouse giveTask(WareHouse wh)
	{
		WorkerLocal[] wks = getWaitingWorkers("Worker");
		System.out.println("wks.length:"+wks.length+";"+wh);
		int total=0;
		System.out.println("第一步");
		wh.setObj("step", 1);//1:group;
		doTaskBatch(wks, wh);
	
	System.out.println("第二步");
		wh.setObj("step", 2);//2:merge;
		doTaskBatch(wks, wh);
System.out.println("第三步");
		wh.setObj("step", 3);//3:sort
		WareHouse[] hmarr = doTaskBatch(wks, wh);
		for(int i=0;i<hmarr.length;i++){
			Object num = hmarr[i].getObj("total");
			if(num!=null)
				total+=(Integer)num;
		}
		wh.setObj("total",total);
		return wh;
	}
	
	public static void main(String[] args)
	{
		Ctor a = new Ctor();
		WareHouse wh = new WareHouse();
		long begin = (new Date()).getTime();
		a.doProject(wh);
		long end = (new Date()).getTime();
		System.out.println("total:"+wh.getObj("total")+",time:"+(end-begin)/1000+"s");
	}
}

附件是生成的一亿个整型随机数的代码

评论

4 楼 evanzzy 2014-08-12  

就是普通整型数组，不用排序，直接按照下标放就可以了。

3 楼 臻是二哥 2014-08-12  

用什么存储的1亿个数据，用那种算法拍的序

2 楼 臻是二哥 2014-08-11  

直接死掉了，你可以试试

1 楼 evanzzy 2014-08-11  

弄一个上亿的整型数组可不可以？直接对号入座就是了，用不了多少内存。