Ticket #14826: 0001-Run-updater-every-hour-depending-on-property-onionoo.patch

File 0001-Run-updater-every-hour-depending-on-property-onionoo.patch, 5.9 KB (added by iwakeh, 5 years ago)
  • src/main/java/org/torproject/onionoo/cron/Main.java

    From dcfee97712572212099ca59edaa8f5c68f87e3d0 Mon Feb 09 17:00:00 2015
    From: iwakeh <iwakeh@users.ourproject.org>
    Date: Mon, 09 Feb 2015 17:00:00 +0000
    Subject: [PATCH] Run updater every hour depending on property
     onionoo.cron.runonce. Defaults to 'true', i.e. running one time.
    
    ---
     .../java/org/torproject/onionoo/cron/Main.java     | 140 +++++++++++++--------
     1 file changed, 88 insertions(+), 52 deletions(-)
    
    diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java
    index 986f22d..0ec8279 100644
    a b  
    11/* Copyright 2011, 2012 The Tor Project
    22 * See LICENSE for licensing information */
    33package org.torproject.onionoo.cron;
    4 
     4import static java.util.concurrent.TimeUnit.MINUTES;
     5import java.util.concurrent.Executors;
     6import java.util.concurrent.ScheduledFuture;
     7import java.util.concurrent.ScheduledExecutorService;
    58import org.slf4j.Logger;
    69import org.slf4j.LoggerFactory;
    710import org.torproject.onionoo.docs.DocumentStore;
    import org.torproject.onionoo.util.LockFile; 
    1316import org.torproject.onionoo.writer.DocumentWriterRunner;
    1417
    1518/* Update search data and status data files. */
     19
    1620public class Main {
    1721
    1822  private static Logger log = LoggerFactory.getLogger(Main.class);
    public class Main { 
    2024  private Main() {
    2125  }
    2226
     27  private static final ScheduledExecutorService scheduler =
     28      Executors.newScheduledThreadPool(1);
     29
    2330  public static void main(String[] args) {
    24     log.debug("Started ...");
    25     LockFile lf = new LockFile();
    26     log.info("Initializing.");
    27     if (lf.acquireLock()) {
    28       log.info("Acquired lock");
    29     } else {
    30       log.error("Could not acquire lock.  Is Onionoo "
    31           + "already running?  Terminating");
     31    boolean runOnce = "true".equals(System.getProperty(
     32        "onionoo.cron.runonce", "true"));
     33    if (runOnce){
     34      log.info("Going to run one-time updater ... ");
     35      new Updater().run();
    3236      return;
    3337    }
    34     log.debug(" ... running .... ");
    35 
    36     DescriptorSource dso = DescriptorSourceFactory.getDescriptorSource();
    37     log.info("Initialized descriptor source");
    38     DocumentStore ds = DocumentStoreFactory.getDocumentStore();
    39     log.info("Initialized document store");
    40     StatusUpdateRunner sur = new StatusUpdateRunner();
    41     log.info("Initialized status update runner");
    42     DocumentWriterRunner dwr = new DocumentWriterRunner();
    43     log.info("Initialized document writer runner");
    44 
    45     log.info("Downloading descriptors.");
    46     dso.downloadDescriptors();
    47 
    48     log.info("Reading descriptors.");
    49     dso.readDescriptors();
    50 
    51     log.info("Updating internal status files.");
    52     sur.updateStatuses();
    53 
    54     log.info("Updating document files.");
    55     dwr.writeDocuments();
    56 
    57     log.info("Shutting down.");
    58     dso.writeHistoryFiles();
    59     log.info("Wrote parse histories");
    60     ds.flushDocumentCache();
    61     log.info("Flushed document cache");
    62 
    63     log.info("Gathering statistics.");
    64     sur.logStatistics();
    65     dwr.logStatistics();
    66     log.info("Descriptor source\n" + dso.getStatsString());
    67     log.info("Document store\n" + ds.getStatsString());
    68 
    69     log.info("Releasing lock.");
    70     if (lf.releaseLock()) {
    71       log.info("Released lock");
    72     } else {
    73       log.error("Could not release lock.  The next "
    74           + "execution may not start as expected");
    75     }
     38    log.info("Periodic updater started.");
     39    final Runnable updater = new Updater();
     40
     41    // run after one minute delay and then
     42    // with a delay of an hour
     43    final ScheduledFuture<?> updaterHandle =
     44    scheduler.scheduleWithFixedDelay(updater, 1, 60, MINUTES);
     45    log.info("Periodic updater will run every hour.");
     46  }
     47
     48  private static class Updater implements Runnable{
     49
     50    private Logger log = LoggerFactory.getLogger(Main.class);
     51
     52    public void run() {
     53      log.debug("Started update ...");
     54      LockFile lf = new LockFile();
     55      log.info("Initializing.");
     56      if (lf.acquireLock()) {
     57          log.info("Acquired lock");
     58      } else {
     59          log.error("Could not acquire lock.  Is Onionoo "
     60                    + "already running?  Terminating");
     61          return;
     62      }
     63      log.debug(" ... running .... ");
     64
     65      DescriptorSource dso = DescriptorSourceFactory.getDescriptorSource();
     66      log.info("Initialized descriptor source");
     67      DocumentStore ds = DocumentStoreFactory.getDocumentStore();
     68      log.info("Initialized document store");
     69      StatusUpdateRunner sur = new StatusUpdateRunner();
    7670
    77     log.info("Terminating.");
     71      log.info("Initialized status update runner");
     72      DocumentWriterRunner dwr = new DocumentWriterRunner();
     73      log.info("Initialized document writer runner");
     74
     75      log.info("Downloading descriptors.");
     76      dso.downloadDescriptors();
     77
     78      log.info("Reading descriptors.");
     79      dso.readDescriptors();
     80
     81      log.info("Updating internal status files.");
     82      sur.updateStatuses();
     83
     84      log.info("Updating document files.");
     85      dwr.writeDocuments();
     86
     87      log.info("Shutting down.");
     88      dso.writeHistoryFiles();
     89      log.info("Wrote parse histories");
     90      ds.flushDocumentCache();
     91      log.info("Flushed document cache");
     92
     93      log.info("Gathering statistics.");
     94      sur.logStatistics();
     95      dwr.logStatistics();
     96      log.info("Descriptor source\n" + dso.getStatsString());
     97      log.info("Document store\n" + ds.getStatsString());
     98
     99      log.info("Releasing lock.");
     100      if (lf.releaseLock()) {
     101        log.info("Released lock");
     102      } else {
     103        log.error("Could not release lock.  The next "
     104            + "execution may not start as expected");
     105      }
     106
     107      log.info("Done.");
     108
     109      // clean-up to prevent oom-exc
     110      ds.invalidateDocumentCache();
     111      DocumentStoreFactory.setDocumentStore(null);
     112      DescriptorSourceFactory.setDescriptorSource(null);
     113    }
    78114  }
    79115}
    80116