From d66f9982fff0eeb450239cbac63d0b326f885e59 Mon Sep 17 00:00:00 2001 From: John Morales Date: Thu, 23 Jan 2014 13:33:18 -0500 Subject: [PATCH 1/5] bugfix: prevent monitor exception spin loop when MongoDB unreachable. --- build.properties | 2 +- .../com/deftlabs/lock/mongo/impl/Monitor.java | 47 +++++++++++++------ 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/build.properties b/build.properties index 10e74e4..132c304 100644 --- a/build.properties +++ b/build.properties @@ -16,4 +16,4 @@ javac.source=1.5 -lib.version=0.1.6 +lib.version=0.1.7 diff --git a/src/main/com/deftlabs/lock/mongo/impl/Monitor.java b/src/main/com/deftlabs/lock/mongo/impl/Monitor.java index 590e146..bc429f4 100644 --- a/src/main/com/deftlabs/lock/mongo/impl/Monitor.java +++ b/src/main/com/deftlabs/lock/mongo/impl/Monitor.java @@ -53,7 +53,7 @@ static class LockHeartbeat extends MonitorThread { } @Override - boolean monitor() throws InterruptedException { + void monitor() { for (final String lockName : _locks.keySet()) { final DistributedLock lock = _locks.get(lockName); @@ -63,8 +63,11 @@ boolean monitor() throws InterruptedException { LockDao.heartbeat(_mongo, lockName, lockId, lock.getOptions(), _svcOptions); } + } - return _shutdown.await(_svcOptions.getHeartbeatFrequency(), TimeUnit.MILLISECONDS); + @Override + long awaitMillis() { + return _svcOptions.getHeartbeatFrequency(); } } @@ -80,9 +83,13 @@ static class LockTimeout extends MonitorThread { } @Override - boolean monitor() throws InterruptedException { + void monitor() { LockDao.expireInactiveLocks(_mongo, _svcOptions); - return _shutdown.await(_svcOptions.getTimeoutFrequency(), TimeUnit.MILLISECONDS); + } + + @Override + long awaitMillis() { + return _svcOptions.getTimeoutFrequency(); } } @@ -100,7 +107,7 @@ static class LockUnlocked extends MonitorThread { } @Override - boolean monitor() throws InterruptedException { + void monitor() { for (final String lockName : _locks.keySet()) { final DistributedLock lock = _locks.get(lockName); @@ -112,8 +119,11 @@ boolean monitor() throws InterruptedException { // The lock is not locked, wakeup any blocking threads. lock.wakeupBlocked(); } + } - return _shutdown.await(_svcOptions.getLockUnlockedFrequency(), TimeUnit.MILLISECONDS); + @Override + long awaitMillis() { + return _svcOptions.getLockUnlockedFrequency(); } } @@ -140,13 +150,18 @@ private static abstract class MonitorThread extends Thread { } @Override public void run() { - boolean shutdown = false; try { - while (!shutdown) { - try { shutdown = monitor(); - } catch (final InterruptedException ie) { break; - } catch (final Throwable t) { LOG.log(Level.SEVERE, t.getMessage(), t); } - } + // do-while to eagerly try at startup for any initial cleanup. + do { + try { + monitor(); + } catch (final Throwable t) { + LOG.log(Level.SEVERE, t.getMessage(), t); + } + } while (!_shutdown.await(awaitMillis(), TimeUnit.MILLISECONDS)); + } catch (InterruptedException ignored) { + // Safe exit. + Thread.currentThread().interrupt(); } finally { _exited.countDown(); } @@ -156,7 +171,9 @@ private static abstract class MonitorThread extends Thread { * Performs check and awaits shutdown signal for configured amount of milliseconds * @return true if shutdown() was called, false otherwise. */ - abstract boolean monitor() throws InterruptedException; + abstract void monitor(); + + abstract long awaitMillis(); void shutdown() throws InterruptedException { _shutdown.countDown(); @@ -168,8 +185,8 @@ void shutdown() throws InterruptedException { final Mongo _mongo; final DistributedLockSvcOptions _svcOptions; final Map _locks; - final CountDownLatch _shutdown; - final CountDownLatch _exited; + private final CountDownLatch _shutdown; + private final CountDownLatch _exited; } private static final Logger LOG = Logger.getLogger("com.deftlabs.lock.mongo.Monitor"); From d04ce784b0f1b892e4d1cbacf2adc5951b9a3b6f Mon Sep 17 00:00:00 2001 From: John Morales Date: Fri, 24 Jan 2014 09:50:50 -0500 Subject: [PATCH 2/5] code review feedback --- src/main/com/deftlabs/lock/mongo/impl/Monitor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/com/deftlabs/lock/mongo/impl/Monitor.java b/src/main/com/deftlabs/lock/mongo/impl/Monitor.java index bc429f4..6d97d41 100644 --- a/src/main/com/deftlabs/lock/mongo/impl/Monitor.java +++ b/src/main/com/deftlabs/lock/mongo/impl/Monitor.java @@ -159,7 +159,7 @@ private static abstract class MonitorThread extends Thread { LOG.log(Level.SEVERE, t.getMessage(), t); } } while (!_shutdown.await(awaitMillis(), TimeUnit.MILLISECONDS)); - } catch (InterruptedException ignored) { + } catch (final InterruptedException ignored) { // Safe exit. Thread.currentThread().interrupt(); } finally { From 6b84000636b523dff86eefed657e75d3d0688dd0 Mon Sep 17 00:00:00 2001 From: John Morales Date: Wed, 26 Feb 2014 18:35:44 -0500 Subject: [PATCH 3/5] Fixing corrupted locking state in case where unlock DB call fails. --- src/main/com/deftlabs/lock/mongo/impl/LockImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java b/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java index eb63fcd..dff34ef 100644 --- a/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java +++ b/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java @@ -179,9 +179,9 @@ private boolean tryDistributedLock() { } @Override public void unlock() { - LockDao.unlock(_mongo, _name, _svcOptions, _lockOptions, _lockId); _locked.set(false); _lockId = null; + LockDao.unlock(_mongo, _name, _svcOptions, _lockOptions, _lockId); LockSupport.unpark(_waitingThreads.peek()); } From 9b5e6da6dc65561a96c8ce345445ebf31e5e0c66 Mon Sep 17 00:00:00 2001 From: John Morales Date: Wed, 26 Feb 2014 18:37:09 -0500 Subject: [PATCH 4/5] Another version bump to capture the unlock fix --- build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.properties b/build.properties index 132c304..d5a5530 100644 --- a/build.properties +++ b/build.properties @@ -16,4 +16,4 @@ javac.source=1.5 -lib.version=0.1.7 +lib.version=0.1.8 From 8726a7096617a031e82c4b7bf09074e819bb3340 Mon Sep 17 00:00:00 2001 From: John Morales Date: Wed, 26 Feb 2014 19:38:51 -0500 Subject: [PATCH 5/5] Fixing lockId durp. --- src/main/com/deftlabs/lock/mongo/impl/LockImpl.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java b/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java index dff34ef..84c4b8e 100644 --- a/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java +++ b/src/main/com/deftlabs/lock/mongo/impl/LockImpl.java @@ -180,9 +180,12 @@ private boolean tryDistributedLock() { @Override public void unlock() { _locked.set(false); - _lockId = null; - LockDao.unlock(_mongo, _name, _svcOptions, _lockOptions, _lockId); - LockSupport.unpark(_waitingThreads.peek()); + try { + LockDao.unlock(_mongo, _name, _svcOptions, _lockOptions, _lockId); + } finally { + _lockId = null; + LockSupport.unpark(_waitingThreads.peek()); + } } /**