Distributed Lock

1 Overview

      在分布式系统中，通常会避免使用分布式锁。然而在某些场景下，还是存在对分布式锁的需求。跟普通锁相比，分布式锁面需要对的问题更多，例如怎样保证某个进程在持有锁时意外终止之后，其它进程也能够正常地获得锁等等。笔者认为一个比较好的分布式锁实现是Terracotta，但是这不是本文的重点，感兴趣的读者可以参考笔者的Terracotta in Action 系列文章（http://whitesock.iteye.com/blog/351780 , http://whitesock.iteye.com/blog/352876 , http://whitesock.iteye.com/blog/354587 ）。

      除了Terracotta，不少其它开源项目也声称支持分布式锁，例如ZooKeeper，JGroups和Hazelcast等。在这些项目中，笔者倾向于使用ZooKeeper。ZooKeeper在其官方文档的ZooKeeper Recipes and Solutions章节中介绍了一个分布式锁的实现，本文主要对该版本进行了改良。关于Hazelcast，笔者不得不说，其官方文档文字不少但却苍白，很多内容介绍的都是浅尝辄止，难道是强迫开发人员去仔细地阅读源码，或者参加其价格不菲的培训？

 

2 Implementation

     首先，笔者希望分布式锁能够支持Java并发包中的Lock接口，并且最好是可重入的。此外，在某个进程持有分布式锁的过程中，如果不能保证该锁不会被其它进程同时持有（例如网络故障），那么至少应该能够通知锁的持有者，以便其采取相应的应对措施。以下是笔者对分布式锁的定义：

import java.util.concurrent.locks.Lock;

public interface DistributedLock extends Lock {
    
    Listener getListener();
    
    void setListener(Listener listener);
    
    /**
     * 
     */
    interface Listener {
        
        void onAbort(DistributedLock lock, Exception e);
    }
}

    其中Listener接口的作用是，在无法排它独占该锁时进行回调。接下来是笔者的两个实现的共通父类。

import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

public abstract class AbstractDistributedLock implements DistributedLock {
    //
    protected volatile boolean verbose;
    protected volatile Listener listener;
    protected final ReentrantLock lock = new ReentrantLock();

    //
    protected abstract void doLock();
    protected abstract void doUnlock();
    protected abstract boolean doTryLock();
    protected abstract void doLockInterruptibly() throws InterruptedException;
    protected abstract boolean doTryLock(long timeout, TimeUnit unit) throws InterruptedException;
    
    /**
     * 
     */
    public boolean isVerbose() {
        return verbose;
    }

    public void setVerbose(boolean verbose) {
        this.verbose = verbose;
    }
    
    public boolean isLocked() {
        return this.lock.isLocked();
    }
    
    public boolean isHeldByCurrentThread() {
        return this.lock.isHeldByCurrentThread();
    }
    
    /**
     * 
     */
    @Override
    public Listener getListener() {
        return this.listener;
    }

    @Override
    public void setListener(Listener listener) {
        this.listener = listener;
    }
    
    /**
     * 
     */
    @Override
    public void lock() {
        //
        this.lock.lock();
        if(this.lock.getHoldCount() > 1) return;
        
        //
        boolean succeed = false;
        try {
            doLock();
            succeed = true;
        } finally {
            if(!succeed) {
                this.lock.unlock();
            }
        }
    }

    @Override
    public void lockInterruptibly() throws InterruptedException {
        //
        this.lock.lockInterruptibly();
        if(this.lock.getHoldCount() > 1) return;
        
        //
        boolean succeed = false;
        try {
            doLockInterruptibly();
            succeed = true;
        } finally {
            if(!succeed) {
                this.lock.unlock();
            }
        }
    }
    
    @Override
    public boolean tryLock() {
        //
        if(!this.lock.tryLock()) return false;
        if(this.lock.getHoldCount() > 1) return true;
        
        //
        boolean succeed = false;
        try {
            succeed = doTryLock();
        } finally {
            if(!succeed) {
                this.lock.unlock();
            }
        }
        return succeed;
    }

    @Override
    public boolean tryLock(long timeout, TimeUnit unit) throws InterruptedException {
        //
        final long mark = System.nanoTime();
        if(!this.lock.tryLock(timeout, unit)) return false;
        if(this.lock.getHoldCount() > 1) return true;
        
        //
        boolean succeed = false;
        try {
            timeout = TimeUnit.NANOSECONDS.convert(timeout, unit) - (System.nanoTime() - mark);
            if(timeout >= 0) {
                succeed = doTryLock(timeout, TimeUnit.NANOSECONDS);
            }
        } finally {
            if(!succeed) {
                this.lock.unlock();
            }
        }
        return succeed;
    }

    @Override
    public void unlock() {
        //
        if(!this.lock.isHeldByCurrentThread()) return;
        if(this.lock.getHoldCount() > 1) return;
        
        //
        try {
            doUnlock();
        } finally {
            this.lock.unlock();
        }
    }
    
    @Override
    public Condition newCondition() {
        throw new UnsupportedOperationException();
    }
}

 

2.1 MySQL Named Lock

    在讨论ZooKeeper的分布式锁实现之前，先介绍一下笔者基于MySQL Named Lock的一个实现。

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;

import javax.sql.DataSource;

import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.apache.commons.lang.exception.NestableRuntimeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public final class MySQLNamedLock extends AbstractDistributedLock {
    //
    private static final Logger LOGGER = LoggerFactory.getLogger(MySQLNamedLock.class);
    
    //
    private String name;
    private DataSource dataSource;
    private long validationInterval = 1000L;
    private ScheduledExecutorService scheduler;
    private final AtomicReference<Connection> connection;
    private final AtomicReference<ScheduledFuture<?>> future;

    /**
     * 
     */
    public MySQLNamedLock() {
        this(null, null, null);
    }
    
    public MySQLNamedLock(String name, DataSource dataSource, ScheduledExecutorService scheduler) {
        this.name = name;
        this.scheduler = scheduler;
        this.dataSource = dataSource;
        this.connection = new AtomicReference<Connection>();
        this.future = new AtomicReference<ScheduledFuture<?>>();
    }
    
    /**
     * 
     */
    @Override
    public String toString() {
        return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
        .append("name", this.name).toString();
    }
    
    /**
     * 
     */
    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }

    public long getValidationInterval() {
        return validationInterval;
    }

    public void setValidationInterval(long interval) {
        this.validationInterval = interval;
    }
    
    public DataSource getDataSource() {
        return dataSource;
    }

    public void setDataSource(DataSource dataSource) {
        this.dataSource = dataSource;
    }
    
    public ScheduledExecutorService getScheduler() {
        return scheduler;
    }

    public void setScheduler(ScheduledExecutorService scheduler) {
        this.scheduler = scheduler;
    }
    
    /**
     * 
     */
    @Override
    protected void doLock() {
        doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);
    }

    @Override
    protected void doLockInterruptibly() {
        doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);
    }

    @Override
    protected boolean doTryLock() {
        return doTryLock(0, TimeUnit.SECONDS);
    }
    
    @Override
    protected boolean doTryLock(long timeout, TimeUnit unit) {
        //
        Integer r = null;
        ResultSet rs = null;
        PreparedStatement ps = null;
        try {
            this.connection.set(this.dataSource.getConnection());
            ps = this.connection.get().prepareStatement("SELECT GET_LOCK(?, ?)");
            ps.setString(1, this.name);
            ps.setInt(2, (int)TimeUnit.SECONDS.convert(timeout, unit));
            rs = ps.executeQuery();
            if(rs.next()) {
                r = rs.getInt(1);
                if(rs.wasNull()) r = null;
            }
        } catch(Exception e) {
            throw new NestableRuntimeException("failed to lock, name: " + this.name, e);
        } finally {
            JdbcUtils.closeQuietly(rs);
            JdbcUtils.closeQuietly(ps);
        }
        
        //
        final boolean succeed = (r != null && r == 1);
        if(succeed && this.listener != null) {
            final long interval = this.validationInterval;
            this.future.set(this.scheduler.scheduleWithFixedDelay(new ValidationTask(), interval, interval, TimeUnit.MILLISECONDS));
        }
        
        //
        return succeed;
    }

    @Override
    protected void doUnlock() {
        //
        final ScheduledFuture<?> f = this.future.getAndSet(null);
        if(f != null) f.cancel(true);
        
        //
        Integer r = null;
        ResultSet rs = null;
        PreparedStatement ps = null;
        try {
            //
            ps = this.connection.get().prepareStatement("SELECT RELEASE_LOCK(?)");
            ps.setString(1, this.name);
            rs = ps.executeQuery();
            if(rs.next()) {
                r = rs.getInt(1);
                if(rs.wasNull()) r = null;
            }
            
            //
            if(r == null) {
                LOGGER.warn("lock does NOT exist, name: {}", this.name);
            } else if(r == 0) {
                LOGGER.warn("lock was NOT accquired by current thread, name: {}", this.name);
            } else {
                LOGGER.warn("failed to unlock, name: {}, result: {}", this.name, r);
            }
        } catch(Exception e) {
            throw new NestableRuntimeException("failed to unlock, name: " + this.name, e);
        } finally {
            JdbcUtils.closeQuietly(rs);
            JdbcUtils.closeQuietly(ps);
            JdbcUtils.closeQuietly(this.connection.getAndSet(null));
        }
    }
    
    /**
     * 
     */
    private class ValidationTask implements Runnable {

        @Override
        public void run() {
            try {
                ((com.mysql.jdbc.Connection)connection.get()).ping();
            } catch(Exception e) {
                //
                if(isLocked() && listener != null && connection.get() != null) {
                    listener.onAbort(MySQLNamedLock.this, e);
                }
                
                //
                throw new NestableRuntimeException(e); // Note: suppress subsequent executions 
            }
        }
    }
}

    需要注意的是，如果在该锁上注册了Listener，并且Connection在持有锁的过程中失效，那么该Listener会被回调。

2.2 ZooKeeper Lock

    以下代码是笔者对ZooKeeper官方版本的改良：

import java.lang.management.ManagementFactory;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.commons.lang.builder.ToStringStyle;
import org.apache.commons.lang.exception.NestableRuntimeException;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public final class ZooKeeperLock extends AbstractDistributedLock {
    //
    private static final Logger LOGGER = LoggerFactory.getLogger(ZooKeeperLock.class);

    //
    private String directory;
    private ZooKeeper zookeeper;
    private final String processName;
    private final AtomicReference<ZooKeeperLocker> locker;

    /**
     * 
     */
    public ZooKeeperLock() {
        this(null, null);
    }

    public ZooKeeperLock(ZooKeeper zookeeper, String directory) {
        this.zookeeper = zookeeper;
        this.directory = directory;
        this.locker = new AtomicReference<ZooKeeperLocker>();
        this.processName = ManagementFactory.getRuntimeMXBean().getName();
    }

    /**
     * 
     */
    @Override
    public String toString() {
        return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
        .append("directory", this.directory).toString();
    }
    
    /**
     * 
     */
    public String getDirectory() {
        return directory;
    }

    public void setDirectory(String directory) {
        this.directory = directory;
    }
    
    public ZooKeeper getZookeeper() {
        return zookeeper;
    }

    public void setZookeeper(ZooKeeper zookeeper) {
        this.zookeeper = zookeeper;
    }

    /**
     * 
     */
    @Override
    protected void doLock() {
        doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);
    }

    @Override
    protected void doLockInterruptibly() {
        doTryLock(Integer.MAX_VALUE, TimeUnit.SECONDS);
    }
    
    @Override
    protected boolean doTryLock() {
        return doTryLock(0, TimeUnit.SECONDS);
    }

    @Override
    protected boolean doTryLock(long timeout, TimeUnit unit) {
        try {
            this.locker.set(new ZooKeeperLocker());
            return this.locker.get().lock(timeout, unit);
        } catch(Exception e) {
            throw new NestableRuntimeException("failed to lock, directory: " + this.directory, e);
        }
    }

    @Override
    protected void doUnlock() {
        try {
            this.locker.get().unlock();
        } catch(Exception e) {
            throw new NestableRuntimeException("failed to unlock, directory: " + this.directory, e);
        } finally {
            this.locker.set(null);
        }
    }

    /**
     * 
     */
    private class ZooKeeperLocker implements Watcher {
        //
        private volatile String name;
        private volatile CountDownLatch latch;

        /**
         * 
         */
        @Override
        public void process(WatchedEvent event) {
            //
            if(this.latch != null) {
                this.latch.countDown();
            }
            
            //
            if(isVerbose() && LOGGER.isInfoEnabled()) {
                LOGGER.info("received an event: {}", event);
            }
        }
        
        public boolean lock(long timeout, TimeUnit unit) throws Exception {
            boolean succeed = false;
            try {
                do {
                    final long mark = System.nanoTime();
                    timeout = TimeUnit.NANOSECONDS.convert(timeout, unit);
                    try {
                        succeed = doLock(timeout, TimeUnit.NANOSECONDS);
                        break;
                    } catch (KeeperException.ConnectionLossException e) {
                        timeout -= (System.nanoTime() - mark);
                        if(isVerbose() && LOGGER.isInfoEnabled()) {
                            LOGGER.info("connection was lost, directory: {}, name: {}, message: {}", new Object[]{directory, this.name, e.getMessage()});
                        }
                    }
                }
                while(timeout > 0);
            } finally {
                if(!succeed) { // Unlock quietly
                    try {
                        unlock();
                    } catch(Exception e) {
                        LOGGER.warn("failed to unlock, directory: " + directory + ", name: " + this.name, e);
                    }
                }
            }
            return succeed;
        }
        
        public void unlock() throws Exception {
            try {
                zookeeper.delete(directory + "/" + this.name, -1);
            } catch (KeeperException.NoNodeException e) {
                LOGGER.warn("node does NOT exist, directory: {}, name: {}, message: {}", new Object[]{directory, this.name, e.getMessage()});
            } finally {
                this.name = null;
            }
        }
        
        /**
         * 
         */
        private Boolean doLock(long timeout, TimeUnit unit) throws Exception {
            boolean succeed = false;
            do {
                //
                final long mark = System.nanoTime();
                timeout = TimeUnit.NANOSECONDS.convert(timeout, unit);

                //
                if (this.name == null) {
                    this.name = findOrCreateChild();
                }
                
                //
                final List<String> children = zookeeper.getChildren(directory, false);
                if (children.isEmpty()) {
                    this.name = null;
                    LOGGER.warn("could not find any child, directory: {}, name: {}", new Object[]{directory, this.name});
                } else {
                    final SequenceComparator comparator = new SequenceComparator();
                    Collections.sort(children, comparator);
                    final int index = Collections.binarySearch(children, this.name, comparator);
                    if (index > 0) { // Not the first one
                        this.latch = new CountDownLatch(1);
                        final String previous = children.get(index - 1);
                        final Stat stat = zookeeper.exists(directory + "/" + previous, this);
                        if (stat != null) {
                            this.latch.await(timeout, TimeUnit.NANOSECONDS);
                            this.latch = null;
                        } else {
                            LOGGER.warn("could not find the previous child, directory: {}, name: {}", new Object[]{directory, this.name});
                        }
                    } else {
                        final String owner = children.get(0);
                        if (this.name != null && owner != null && this.name.equals(owner)) {
                            succeed = true;
                        } else {
                            LOGGER.warn("the lock should be held by current thread, directory: {}, name: {}, owner: {}", new Object[]{directory, this.name, owner});
                        }
                    }
                }
                
                //
                timeout -= (System.nanoTime() - mark);
            } while (!succeed && timeout >= 0);
            return succeed;
        }
        
        private String findOrCreateChild() throws Exception {
            //
            final String prefix = zookeeper.getSessionId() + "-";
            final List<String> children = zookeeper.getChildren(directory, false);
            for (String child : children) {
                if (child.startsWith(prefix)) {
                    if(isVerbose() && LOGGER.isInfoEnabled()) {
                        LOGGER.info("found a child, directory: {}, child: {}", new Object[]{directory, child});
                    }
                    return child;
                }
            }
            
            //
            final String data = Thread.currentThread().getId() + "@" + processName;
            final String path = zookeeper.create(directory + "/" + prefix, data.getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL_SEQUENTIAL);
            final String child = path.substring(path.lastIndexOf("/") + 1);
            if(isVerbose() && LOGGER.isInfoEnabled()) {
                LOGGER.info("created a child, directory: {}, path: {}", new Object[]{directory, child});
            }
            return child;
        }
    }
    
    /**
     * 
     */
    private static class SequenceComparator implements Comparator<String> {

        @Override
        public int compare(String lhs, String rhs) {
            final int index1 = lhs.lastIndexOf('-');
            final int index2 = rhs.lastIndexOf('-');
            final int sequence1 = Integer.parseInt(lhs.substring(index1 + 1));
            final int sequence2 = Integer.parseInt(rhs.substring(index2 + 1));
            return sequence1 - sequence2;
        }
    }
}

    ZooKeeperLock是fair的，并且在Node中保存的数据是线程ID，进程ID以及主机名。需要注意的是，应该为ZooKeeper部署集群，此外还需要保证传入ZooKeeperLock构造函数中的ZooKepper实例已经跟Server建立的连接，否则zookeeper.getSessionId()会返回0，从而导致错误。

 

3 disclaimer

    笔者只对以上代码进行了简单的测试，因此可能存在错误，请慎重使用。如果发现问题，感谢反馈。

评论

4 楼 liguanqun811 2015-02-10  

不知道楼主是否对zookeeper实现的分布式锁进行过性能测试？我测试下了，感觉差劲。http://liguanqun811.iteye.com/blog/2184566

3 楼 hobitton 2015-02-02  

mysql的get lock有版本限制，否则get lock可能出现的效果和想象的不一样，见：
http://dev.mysql.com/doc/refman/5.7/en/miscellaneous-functions.html#function_get-lock

5.7.5之前的解决方案真实一个蛋疼的解决方案。

2 楼 fandayrockworld 2012-09-11  

引用

关于Hazelcast，笔者不得不说，其官方文档文字不少但却苍白，很多内容介绍的都是浅尝辄止，难道是强迫开发人员去仔细地阅读源码，或者参加其价格不菲的培训？

上面的观点不认同，他的文档我觉得还是写的很好的，通俗易懂，而且把该说的都说了。 http://www.hazelcast.com/docs/2.3/manual/pdf/hazelcast-documentation.pdf

1 楼 fireflyk 2012-04-21  

写的很好！也做了个分布式锁，用的非持久化的内存数据库，在思考宕机情况下，有什么好的解决方案