001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.hawtdb;
018
019 import java.io.File;
020 import java.io.IOException;
021 import java.util.Collections;
022 import java.util.Iterator;
023 import java.util.LinkedHashSet;
024 import java.util.Map;
025 import java.util.Set;
026 import java.util.concurrent.TimeUnit;
027
028 import org.apache.camel.CamelContext;
029 import org.apache.camel.Exchange;
030 import org.apache.camel.impl.ServiceSupport;
031 import org.apache.camel.spi.RecoverableAggregationRepository;
032 import org.apache.camel.util.ObjectHelper;
033 import org.apache.camel.util.ServiceHelper;
034 import org.apache.commons.logging.Log;
035 import org.apache.commons.logging.LogFactory;
036 import org.fusesource.hawtbuf.Buffer;
037 import org.fusesource.hawtdb.api.SortedIndex;
038 import org.fusesource.hawtdb.api.Transaction;
039
040 /**
041 * An instance of AggregationRepository which is backed by a HawtDB.
042 */
043 public class HawtDBAggregationRepository extends ServiceSupport implements RecoverableAggregationRepository {
044
045 private static final transient Log LOG = LogFactory.getLog(HawtDBAggregationRepository.class);
046 private HawtDBFile hawtDBFile;
047 private String persistentFileName;
048 private String repositoryName;
049 private int bufferSize = 8 * 1024 * 1024;
050 private boolean sync = true;
051 private short pageSize = 512;
052 private boolean returnOldExchange;
053 private HawtDBCamelCodec codec = new HawtDBCamelCodec();
054 private long recoveryInterval = 5000;
055 private boolean useRecovery = true;
056 private int maximumRedeliveries;
057 private String deadLetterUri;
058
059 /**
060 * Creates an aggregation repository
061 */
062 public HawtDBAggregationRepository() {
063 }
064
065 /**
066 * Creates an aggregation repository
067 *
068 * @param repositoryName the repository name
069 */
070 public HawtDBAggregationRepository(String repositoryName) {
071 ObjectHelper.notEmpty(repositoryName, "repositoryName");
072 this.repositoryName = repositoryName;
073 }
074
075 /**
076 * Creates an aggregation repository using a new {@link org.apache.camel.component.hawtdb.HawtDBFile}
077 * that persists using the provided file.
078 *
079 * @param repositoryName the repository name
080 * @param persistentFileName the persistent store filename
081 */
082 public HawtDBAggregationRepository(String repositoryName, String persistentFileName) {
083 ObjectHelper.notEmpty(repositoryName, "repositoryName");
084 ObjectHelper.notEmpty(persistentFileName, "persistentFileName");
085 this.repositoryName = repositoryName;
086 this.persistentFileName = persistentFileName;
087 }
088
089 /**
090 * Creates an aggregation repository using the provided {@link org.apache.camel.component.hawtdb.HawtDBFile}.
091 *
092 * @param repositoryName the repository name
093 * @param hawtDBFile the hawtdb file to use as persistent store
094 */
095 public HawtDBAggregationRepository(String repositoryName, HawtDBFile hawtDBFile) {
096 ObjectHelper.notEmpty(repositoryName, "repositoryName");
097 ObjectHelper.notNull(hawtDBFile, "hawtDBFile");
098 this.hawtDBFile = hawtDBFile;
099 this.repositoryName = repositoryName;
100 }
101
102 public Exchange add(final CamelContext camelContext, final String key, final Exchange exchange) {
103 if (LOG.isDebugEnabled()) {
104 LOG.debug("Adding key [" + key + "] -> " + exchange);
105 }
106 try {
107 // If we could guarantee that the key and exchange are immutable,
108 // then we could have stuck them directly into the index,
109 // HawtDB could then eliminate the need to marshal and un-marshal
110 // in some cases. But since we can't.. we are going to force
111 // early marshaling.
112 final Buffer keyBuffer = codec.marshallKey(key);
113 final Buffer exchangeBuffer = codec.marshallExchange(camelContext, exchange);
114 Buffer rc = hawtDBFile.execute(new Work<Buffer>() {
115 public Buffer execute(Transaction tx) {
116 SortedIndex<Buffer, Buffer> index = hawtDBFile.getRepositoryIndex(tx, repositoryName, true);
117 return index.put(keyBuffer, exchangeBuffer);
118 }
119
120 @Override
121 public String toString() {
122 return "Adding key [" + key + "]";
123 }
124 });
125 if (rc == null) {
126 return null;
127 }
128
129 // only return old exchange if enabled
130 if (isReturnOldExchange()) {
131 return codec.unmarshallExchange(camelContext, rc);
132 }
133 } catch (IOException e) {
134 throw new RuntimeException("Error adding to repository " + repositoryName + " with key " + key, e);
135 }
136
137 return null;
138 }
139
140 public Exchange get(final CamelContext camelContext, final String key) {
141 Exchange answer = null;
142 try {
143 final Buffer keyBuffer = codec.marshallKey(key);
144 Buffer rc = hawtDBFile.execute(new Work<Buffer>() {
145 public Buffer execute(Transaction tx) {
146 SortedIndex<Buffer, Buffer> index = hawtDBFile.getRepositoryIndex(tx, repositoryName, false);
147 if (index == null) {
148 return null;
149 }
150 return index.get(keyBuffer);
151 }
152
153 @Override
154 public String toString() {
155 return "Getting key [" + key + "]";
156 }
157 });
158 if (rc != null) {
159 answer = codec.unmarshallExchange(camelContext, rc);
160 }
161 } catch (IOException e) {
162 throw new RuntimeException("Error getting key " + key + " from repository " + repositoryName, e);
163 }
164
165 if (LOG.isDebugEnabled()) {
166 LOG.debug("Getting key [" + key + "] -> " + answer);
167 }
168 return answer;
169 }
170
171 public void remove(final CamelContext camelContext, final String key, final Exchange exchange) {
172 if (LOG.isDebugEnabled()) {
173 LOG.debug("Removing key [" + key + "]");
174 }
175 try {
176 final Buffer keyBuffer = codec.marshallKey(key);
177 final Buffer confirmKeyBuffer = codec.marshallKey(exchange.getExchangeId());
178 final Buffer exchangeBuffer = codec.marshallExchange(camelContext, exchange);
179 hawtDBFile.execute(new Work<Buffer>() {
180 public Buffer execute(Transaction tx) {
181 SortedIndex<Buffer, Buffer> index = hawtDBFile.getRepositoryIndex(tx, repositoryName, true);
182 // remove from the in progress index
183 index.remove(keyBuffer);
184
185 // and add it to the confirmed index
186 SortedIndex<Buffer, Buffer> indexCompleted = hawtDBFile.getRepositoryIndex(tx, getRepositoryNameCompleted(), true);
187 indexCompleted.put(confirmKeyBuffer, exchangeBuffer);
188 return null;
189 }
190
191 @Override
192 public String toString() {
193 return "Removing key [" + key + "]";
194 }
195 });
196
197 } catch (IOException e) {
198 throw new RuntimeException("Error removing key " + key + " from repository " + repositoryName, e);
199 }
200 }
201
202 public void confirm(final CamelContext camelContext, final String exchangeId) {
203 if (LOG.isDebugEnabled()) {
204 LOG.debug("Confirming exchangeId [" + exchangeId + "]");
205 }
206 try {
207 final Buffer confirmKeyBuffer = codec.marshallKey(exchangeId);
208 hawtDBFile.execute(new Work<Buffer>() {
209 public Buffer execute(Transaction tx) {
210 SortedIndex<Buffer, Buffer> indexCompleted = hawtDBFile.getRepositoryIndex(tx, getRepositoryNameCompleted(), true);
211 return indexCompleted.remove(confirmKeyBuffer);
212 }
213
214 @Override
215 public String toString() {
216 return "Confirming exchangeId [" + exchangeId + "]";
217 }
218 });
219
220 } catch (IOException e) {
221 throw new RuntimeException("Error confirming exchangeId " + exchangeId + " from repository " + repositoryName, e);
222 }
223 }
224
225 public Set<String> getKeys() {
226 final Set<String> keys = new LinkedHashSet<String>();
227
228 hawtDBFile.execute(new Work<Buffer>() {
229 public Buffer execute(Transaction tx) {
230 // interval task could potentially be running while we are shutting down so check for that
231 if (!isRunAllowed()) {
232 return null;
233 }
234
235 SortedIndex<Buffer, Buffer> index = hawtDBFile.getRepositoryIndex(tx, repositoryName, false);
236 if (index == null) {
237 return null;
238 }
239
240 Iterator<Map.Entry<Buffer, Buffer>> it = index.iterator();
241 // scan could potentially be running while we are shutting down so check for that
242 while (it.hasNext() && isRunAllowed()) {
243 Map.Entry<Buffer, Buffer> entry = it.next();
244 Buffer keyBuffer = entry.getKey();
245
246 String key;
247 try {
248 key = codec.unmarshallKey(keyBuffer);
249 } catch (IOException e) {
250 throw new RuntimeException("Error unmarshalling key: " + keyBuffer, e);
251 }
252 if (key != null) {
253 if (LOG.isTraceEnabled()) {
254 LOG.trace("getKey [" + key + "]");
255 }
256 keys.add(key);
257 }
258 }
259 return null;
260
261 }
262
263 @Override
264 public String toString() {
265 return "getKeys";
266 }
267 });
268
269 return Collections.unmodifiableSet(keys);
270 }
271
272 public Set<String> scan(CamelContext camelContext) {
273 final Set<String> answer = new LinkedHashSet<String>();
274 hawtDBFile.execute(new Work<Buffer>() {
275 public Buffer execute(Transaction tx) {
276 // scan could potentially be running while we are shutting down so check for that
277 if (!isRunAllowed()) {
278 return null;
279 }
280
281 SortedIndex<Buffer, Buffer> indexCompleted = hawtDBFile.getRepositoryIndex(tx, getRepositoryNameCompleted(), false);
282 if (indexCompleted == null) {
283 return null;
284 }
285
286 Iterator<Map.Entry<Buffer, Buffer>> it = indexCompleted.iterator();
287 // scan could potentially be running while we are shutting down so check for that
288 while (it.hasNext() && isRunAllowed()) {
289 Map.Entry<Buffer, Buffer> entry = it.next();
290 Buffer keyBuffer = entry.getKey();
291
292 String exchangeId;
293 try {
294 exchangeId = codec.unmarshallKey(keyBuffer);
295 } catch (IOException e) {
296 throw new RuntimeException("Error unmarshalling confirm key: " + keyBuffer, e);
297 }
298 if (exchangeId != null) {
299 if (LOG.isTraceEnabled()) {
300 LOG.trace("Scan exchangeId [" + exchangeId + "]");
301 }
302 answer.add(exchangeId);
303 }
304 }
305 return null;
306 }
307
308 @Override
309 public String toString() {
310 return "Scan";
311 }
312 });
313
314 if (answer.size() == 0) {
315 LOG.trace("Scanned and found no exchange to recover.");
316 } else {
317 if (LOG.isDebugEnabled()) {
318 LOG.debug("Scanned and found " + answer.size() + " exchange(s) to recover (note some of them may already be in progress).");
319 }
320 }
321 return answer;
322
323 }
324
325 public Exchange recover(CamelContext camelContext, final String exchangeId) {
326 Exchange answer = null;
327 try {
328 final Buffer confirmKeyBuffer = codec.marshallKey(exchangeId);
329 Buffer rc = hawtDBFile.execute(new Work<Buffer>() {
330 public Buffer execute(Transaction tx) {
331 SortedIndex<Buffer, Buffer> indexCompleted = hawtDBFile.getRepositoryIndex(tx, getRepositoryNameCompleted(), false);
332 if (indexCompleted == null) {
333 return null;
334 }
335 return indexCompleted.get(confirmKeyBuffer);
336 }
337
338 @Override
339 public String toString() {
340 return "Recovering exchangeId [" + exchangeId + "]";
341 }
342 });
343 if (rc != null) {
344 answer = codec.unmarshallExchange(camelContext, rc);
345 }
346 } catch (IOException e) {
347 throw new RuntimeException("Error recovering exchangeId " + exchangeId + " from repository " + repositoryName, e);
348 }
349
350 if (LOG.isDebugEnabled()) {
351 LOG.debug("Recovering exchangeId [" + exchangeId + "] -> " + answer);
352 }
353 return answer;
354 }
355
356 private int size(final String repositoryName) {
357 int answer = hawtDBFile.execute(new Work<Integer>() {
358 public Integer execute(Transaction tx) {
359 SortedIndex<Buffer, Buffer> index = hawtDBFile.getRepositoryIndex(tx, repositoryName, false);
360 return index != null ? index.size() : 0;
361 }
362
363 @Override
364 public String toString() {
365 return "Size[" + repositoryName + "]";
366 }
367 });
368
369 if (LOG.isDebugEnabled()) {
370 LOG.debug("Size of repository [" + repositoryName + "] -> " + answer);
371 }
372 return answer;
373 }
374
375 public HawtDBFile getHawtDBFile() {
376 return hawtDBFile;
377 }
378
379 public void setHawtDBFile(HawtDBFile hawtDBFile) {
380 this.hawtDBFile = hawtDBFile;
381 }
382
383 public String getRepositoryName() {
384 return repositoryName;
385 }
386
387 private String getRepositoryNameCompleted() {
388 return repositoryName + "-completed";
389 }
390
391 public void setRepositoryName(String repositoryName) {
392 this.repositoryName = repositoryName;
393 }
394
395 public String getPersistentFileName() {
396 return persistentFileName;
397 }
398
399 public void setPersistentFileName(String persistentFileName) {
400 this.persistentFileName = persistentFileName;
401 }
402
403 public boolean isSync() {
404 return sync;
405 }
406
407 public void setSync(boolean sync) {
408 this.sync = sync;
409 }
410
411 public Integer getBufferSize() {
412 return bufferSize;
413 }
414
415 public void setBufferSize(Integer bufferSize) {
416 this.bufferSize = bufferSize;
417 }
418
419 public boolean isReturnOldExchange() {
420 return returnOldExchange;
421 }
422
423 public void setReturnOldExchange(boolean returnOldExchange) {
424 this.returnOldExchange = returnOldExchange;
425 }
426
427 public void setRecoveryInterval(long interval, TimeUnit timeUnit) {
428 this.recoveryInterval = timeUnit.toMillis(interval);
429 }
430
431 public void setRecoveryInterval(long interval) {
432 this.recoveryInterval = interval;
433 }
434
435 public long getRecoveryIntervalInMillis() {
436 return recoveryInterval;
437 }
438
439 public boolean isUseRecovery() {
440 return useRecovery;
441 }
442
443 public void setUseRecovery(boolean useRecovery) {
444 this.useRecovery = useRecovery;
445 }
446
447 public int getMaximumRedeliveries() {
448 return maximumRedeliveries;
449 }
450
451 public void setMaximumRedeliveries(int maximumRedeliveries) {
452 this.maximumRedeliveries = maximumRedeliveries;
453 }
454
455 public String getDeadLetterUri() {
456 return deadLetterUri;
457 }
458
459 public void setDeadLetterUri(String deadLetterUri) {
460 this.deadLetterUri = deadLetterUri;
461 }
462
463 public short getPageSize() {
464 return pageSize;
465 }
466
467 public void setPageSize(short pageSize) {
468 this.pageSize = pageSize;
469 }
470
471 @Override
472 protected void doStart() throws Exception {
473 // either we have a HawtDB configured or we use a provided fileName
474 if (hawtDBFile == null && persistentFileName != null) {
475 hawtDBFile = new HawtDBFile();
476 hawtDBFile.setFile(new File(persistentFileName));
477 hawtDBFile.setSync(isSync());
478 if (getBufferSize() != null) {
479 hawtDBFile.setMappingSegementSize(getBufferSize());
480 }
481 if (getPageSize() > 0) {
482 hawtDBFile.setPageSize(getPageSize());
483 }
484 }
485
486 ObjectHelper.notNull(hawtDBFile, "Either set a persistentFileName or a hawtDBFile");
487 ObjectHelper.notNull(repositoryName, "repositoryName");
488
489 ServiceHelper.startService(hawtDBFile);
490
491 // log number of existing exchanges
492 int current = size(getRepositoryName());
493 int completed = size(getRepositoryNameCompleted());
494
495 if (current > 0) {
496 LOG.info("On startup there are " + current + " aggregate exchanges (not completed) in repository: " + getRepositoryName());
497 } else {
498 LOG.info("On startup there are no existing aggregate exchanges (not completed) in repository: " + getRepositoryName());
499 }
500 if (completed > 0) {
501 LOG.warn("On startup there are " + completed + " completed exchanges to be recovered in repository: " + getRepositoryNameCompleted());
502 } else {
503 LOG.info("On startup there are no completed exchanges to be recovered in repository: " + getRepositoryNameCompleted());
504 }
505 }
506
507 @Override
508 protected void doStop() throws Exception {
509 ServiceHelper.stopService(hawtDBFile);
510 }
511
512 }