Changeset 277:6a9619c93177

Timestamp:

09/03/11 16:52:50 (5 months ago)

Author:

delmitz

Branch:

default

Convert:

svn:7c3792e6-d75b-4784-96a6-b298f655ee64/trunk@2711

Message:

added check disk lack.

Location:

kraken-logstorage/src/main/java/org/krakenapps/logstorage

Files:

• DiskLackAction.java (added)
• DiskSpaceType.java (added)
• LogStorage.java (modified) (1 diff)
• engine/Constants.java (modified) (2 diffs)
• engine/LogStorageEngine.java (modified) (10 diffs)
• query/parser/SearchParser.java (modified) (1 diff)
• script/LogStorageScript.java (modified) (2 diffs)

kraken-logstorage/src/main/java/org/krakenapps/logstorage/LogStorage.java

@@ -43 +43 @@
         void write(Collection<Log> logs);
 
+        int getMinFreeSpaceValue();
+
+        DiskSpaceType getMinFreeSpaceType();
+
+        void setMinFreeSpace(int value, DiskSpaceType type);
+
+        DiskLackAction getDiskLackAction();
+
+        void setDiskLackAction(DiskLackAction action);
+
         Log getLog(LogKey logKey);
 

kraken-logstorage/src/main/java/org/krakenapps/logstorage/engine/Constants.java

@@ -19 +19 @@
         LogMaxIdleTime("log_max_idle_time"), 
         LogFlushInterval("log_flush_interval"), 
-        LogMaxBuffering("log_max_buffering");
+        LogMaxBuffering("log_max_buffering"),
+        MinFreeDiskSpaceType("min_free_disk_space_type", "string"),
+        MinFreeDiskSpaceValue("min_free_disk_space_value"),
+        DiskLackAction("disk_lack_action", "string");
 
         Constants(String name) {
@@ -36 +39 @@
                 return name;
         }
-        
+
         public String getType() {
                 return type;

kraken-logstorage/src/main/java/org/krakenapps/logstorage/engine/LogStorageEngine.java

@@ -41 +41 @@
 import org.apache.felix.ipojo.annotations.Validate;
 import org.krakenapps.codec.EncodingRule;
+import org.krakenapps.logstorage.DiskLackAction;
+import org.krakenapps.logstorage.DiskSpaceType;
 import org.krakenapps.logstorage.Log;
 import org.krakenapps.logstorage.LogCallback;
@@ -62 +64 @@
         private static final int DEFAULT_MAX_LOG_BUFFERING = 10000;
         private static final int DEFAULT_LOG_FLUSH_INTERVAL = 3600000;
+        private static final String DEFAULT_MIN_FREE_SPACE_TYPE = DiskSpaceType.Percentage.toString();
+        private static final int DEFAULT_MIN_FREE_SPACE_VALUE = 10;
+        private static final String DEFAULT_DISK_LACK_ACTION = DiskLackAction.StopLogging.toString();
 
         private LogStorageStatus status = LogStorageStatus.Closed;
@@ -80 +85 @@
         private Thread writerSweeperThread;
 
+        private static final File logDir = new File(System.getProperty("kraken.data.dir"), "kraken-logstorage/log");
+        private DiskSpaceType minFreeSpaceType;
+        private int minFreeSpaceValue;
+        private DiskLackAction diskLackAction;
+
+        static {
+                logDir.mkdirs();
+        }
+
         public LogStorageEngine() {
                 int maxIdleTime = getIntParameter(Constants.LogMaxIdleTime, DEFAULT_MAX_IDLE_TIME);
@@ -87 +101 @@
                 writerSweeper = new WriterSweeper(maxIdleTime, flushInterval);
                 callbacks = new CopyOnWriteArraySet<LogCallback>();
+                minFreeSpaceType = DiskSpaceType.valueOf(getStringParameter(Constants.MinFreeDiskSpaceType,
+                                DEFAULT_MIN_FREE_SPACE_TYPE));
+                minFreeSpaceValue = getIntParameter(Constants.MinFreeDiskSpaceValue, DEFAULT_MIN_FREE_SPACE_VALUE);
+                diskLackAction = DiskLackAction.valueOf(getStringParameter(Constants.DiskLackAction, DEFAULT_DISK_LACK_ACTION));
+        }
+
+        private String getStringParameter(Constants key, String defaultValue) {
+                String value = ConfigUtil.get(prefsvc, key);
+                if (value != null)
+                        return value;
+                return defaultValue;
         }
 
@@ -229 +254 @@
                                         dates.add(dateFormat.parse(file.getName().split("\\.")[0]));
                                 } catch (ParseException e) {
-                                        e.printStackTrace();
-                                }
-                        }
-                }
-
-                Collections.sort(dates, new Comparator<Date>() {
-                        @Override
-                        public int compare(Date o1, Date o2) {
-                                return (int) (o2.getTime() - o1.getTime());
-                        }
-                });
+                                        logger.error("kraken logstorage: invalid log filename, table {}, {}", tableName, file.getName());
+                                }
+                        }
+                }
+
+                Collections.sort(dates, Collections.reverseOrder());
 
                 return dates;
@@ -295 +315 @@
                 LogRecord logdata = new LogRecord(log.getDate(), log.getId(), bb);
                 return logdata;
+        }
+
+        @Override
+        public int getMinFreeSpaceValue() {
+                return minFreeSpaceValue;
+        }
+
+        @Override
+        public DiskSpaceType getMinFreeSpaceType() {
+                return minFreeSpaceType;
+        }
+
+        @Override
+        public void setMinFreeSpace(int value, DiskSpaceType type) {
+                if (type == DiskSpaceType.Percentage) {
+                        if (value <= 0 || value >= 100)
+                                throw new IllegalArgumentException("invalid value");
+                } else if (type == DiskSpaceType.Megabyte) {
+                        if (value <= 0)
+                                throw new IllegalArgumentException("invalid value");
+                } else if (type == null)
+                        throw new IllegalArgumentException("type cannot be null");
+
+                this.minFreeSpaceType = type;
+                this.minFreeSpaceValue = value;
+
+                ConfigUtil.set(prefsvc, Constants.MinFreeDiskSpaceType, type.toString());
+                ConfigUtil.set(prefsvc, Constants.MinFreeDiskSpaceValue, Integer.toString(value));
+        }
+
+        @Override
+        public DiskLackAction getDiskLackAction() {
+                return diskLackAction;
+        }
+
+        @Override
+        public void setDiskLackAction(DiskLackAction action) {
+                if (action == null)
+                        throw new IllegalArgumentException("action cannot be null");
+
+                this.diskLackAction = action;
+
+                ConfigUtil.set(prefsvc, Constants.DiskLackAction, action.toString());
         }
 
@@ -593 +656 @@
         @Override
         public void reload() {
-                int flushInterval = Integer.valueOf(ConfigUtil.get(prefsvc, Constants.LogFlushInterval));
-                int maxIdleTime = Integer.valueOf(ConfigUtil.get(prefsvc, Constants.LogMaxIdleTime));
+                int flushInterval = getIntParameter(Constants.LogFlushInterval, DEFAULT_LOG_FLUSH_INTERVAL);
+                int maxIdleTime = getIntParameter(Constants.LogMaxIdleTime, DEFAULT_MAX_IDLE_TIME);
                 writerSweeper.setFlushInterval(flushInterval);
                 writerSweeper.setMaxIdleTime(maxIdleTime);
+
+                minFreeSpaceType = DiskSpaceType.valueOf(getStringParameter(Constants.MinFreeDiskSpaceType,
+                                DEFAULT_MIN_FREE_SPACE_TYPE));
+                minFreeSpaceValue = getIntParameter(Constants.MinFreeDiskSpaceValue, DEFAULT_MIN_FREE_SPACE_VALUE);
+                diskLackAction = DiskLackAction.valueOf(getStringParameter(Constants.DiskLackAction, DEFAULT_DISK_LACK_ACTION));
         }
 
@@ -618 +686 @@
                 if (status != LogStorageStatus.Open)
                         throw new IllegalStateException("archive not opened");
+        }
+
+        private class LogFile {
+                private String tableName;
+                private Date date;
+                private File index;
+                private File data;
+
+                private LogFile(String tableName, Date date) {
+                        this.tableName = tableName;
+                        this.date = date;
+                        int tableId = tableRegistry.getTableId(tableName);
+                        File tableDir = getTableDirectory(tableId);
+                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+                        this.index = new File(tableDir, sdf.format(date) + ".idx");
+                        this.data = new File(tableDir, sdf.format(date) + ".dat");
+                }
+
+                public void remove() {
+                        index.delete();
+                        data.delete();
+                }
+        }
+
+        private class LogFileComparator implements Comparator<LogFile> {
+                @Override
+                public int compare(LogFile o1, LogFile o2) {
+                        return o1.date.compareTo(o2.date);
+                }
         }
 
@@ -652 +749 @@
 
                                 while (true) {
+                                        if (isDiskLack()) {
+                                                logger.warn("kraken logstorage: not enough disk space");
+                                                if (diskLackAction == DiskLackAction.StopLogging) {
+                                                        logger.info("kraken logstorage: stop logging");
+                                                        stop();
+                                                } else if (diskLackAction == DiskLackAction.RemoveOldLog) {
+                                                        List<LogFile> files = new ArrayList<LogFile>();
+                                                        for (String tableName : tableRegistry.getTableNames()) {
+                                                                for (Date date : getLogDates(tableName))
+                                                                        files.add(new LogFile(tableName, date));
+                                                        }
+                                                        Collections.sort(files, new LogFileComparator());
+                                                        int index = 0;
+                                                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+                                                        do {
+                                                                if (index >= files.size()) {
+                                                                        logger.info("kraken logstorage: stop logging");
+                                                                        stop();
+                                                                        break;
+                                                                }
+                                                                LogFile lf = files.get(index++);
+                                                                logger.info("kraken logstorage: remove old log, table {}, {}", lf.tableName,
+                                                                                sdf.format(lf.date));
+                                                                lf.remove();
+                                                        } while (isDiskLack());
+                                                }
+                                        }
+
                                         try {
                                                 if (doStop)
@@ -707 +832 @@
                         }
                 }
+
+                private boolean isDiskLack() {
+                        long usable = logDir.getUsableSpace();
+                        long total = logDir.getTotalSpace();
+
+                        logger.trace("kraken logstorage: check disk lack, {} {}", minFreeSpaceValue, minFreeSpaceType);
+                        if (minFreeSpaceType == DiskSpaceType.Percentage) {
+                                int percent = (int) (usable * 100 / total);
+                                if (percent < minFreeSpaceValue) {
+                                        logger.warn("kraken logstorage: setted minimum free space {}%, now free space {}%",
+                                                        minFreeSpaceValue, percent);
+                                        return true;
+                                }
+                        } else if (minFreeSpaceType == DiskSpaceType.Megabyte) {
+                                int mega = (int) (usable / 1048576);
+                                if (mega < minFreeSpaceValue) {
+                                        logger.warn("kraken logstorage: setted minimum free space {} MB, now free space {} MB",
+                                                        minFreeSpaceValue, mega);
+                                        return true;
+                                }
+                        }
+
+                        return false;
+                }
         }
 }

kraken-logstorage/src/main/java/org/krakenapps/logstorage/query/parser/SearchParser.java

@@ -33 +33 @@
         public void addSyntax(Syntax syntax) {
                 // @formatter:off
-                syntax.add("search", this, k("eval"), ref("option"), rule(new StringPlaceholder(), 
+                syntax.add("search", this, k("search"), ref("option"), rule(new StringPlaceholder(), 
                                 choice(k("=="), k("!="), k(">"), k("<"), k(">="), k("<="), k("contain"), k("regexp"), k("in")), 
                                 new StringPlaceholder(new char[] {})));

kraken-logstorage/src/main/java/org/krakenapps/logstorage/script/LogStorageScript.java

@@ -92 +92 @@
         }
 
+        public void reload(String[] args) {
+                storage.reload();
+        }
+
         @ScriptUsage(description = "create new table", arguments = { @ScriptArgument(name = "name", type = "string",
                         description = "log table name") })
@@ -202 +206 @@
         }
 
-        @ScriptUsage(description = "set parameters")
+        @ScriptUsage(description = "set parameters", arguments = {
+                        @ScriptArgument(name = "key", type = "string", description = "parameter key"),
+                        @ScriptArgument(name = "value", type = "string", description = "parameter value") })
         public void setParameter(String[] args) {
                 Constants configKey = Constants.parse(args[0]);