Changeset 848:d8b3aadce143

Timestamp:

02/04/12 16:55:30 (4 months ago)

Author:

xeraph

Branch:

default

Message:

• removed main bottleneck of manifest-read (slow down caused by many array manipulation)
• added manual manifestId parsing from change log binary for speed-up
• removed writer.sync() per config add
• getManifest() fast-path for ensureCollection() speed-up (skip config fetch)
• manual manifestId parsing from changeset log for ensureCollection() speed-up

Location:

kraken-confdb/src/main/java/org/krakenapps/confdb

Files:

• ConfigEntry.java (modified) (2 diffs)
• file/ChangeLog.java (modified) (1 diff)
• file/FileConfigCollection.java (modified) (3 diffs)
• file/FileConfigDatabase.java (modified) (5 diffs)
• file/FileConfigIterator.java (modified) (1 diff)
• file/FileManifest.java (modified) (10 diffs)

kraken-confdb/src/main/java/org/krakenapps/confdb/ConfigEntry.java

@@ -36 +36 @@
         }
 
-        @Override
-        public int hashCode() {
-                final int prime = 31;
-                int result = 1;
-                result = prime * result + colId;
-                result = prime * result + docId;
-                return result;
-        }
-
-        /**
-         * the key is composition of collection id and doc id
-         */
-        @Override
-        public boolean equals(Object obj) {
-                if (this == obj)
-                        return true;
-                if (obj == null)
-                        return false;
-                if (getClass() != obj.getClass())
-                        return false;
-                ConfigEntry other = (ConfigEntry) obj;
-                if (colId != other.colId)
-                        return false;
-                if (docId != other.docId)
-                        return false;
-                return true;
-        }
-
         public int getColId() {
                 return colId;
@@ -92 +64 @@
                 return "col=" + colId + ", doc=" + docId + ", rev=" + rev;
         }
-
 }

kraken-confdb/src/main/java/org/krakenapps/confdb/file/ChangeLog.java

@@ -147 +147 @@
                 }
                 return m;
+        }
+
+        /**
+         * get manifest id from change log binary. manual parsing for speed-up
+         */
+        public static int getManifest(byte[] b) {
+                ByteBuffer bb = ByteBuffer.wrap(b);
+                bb.get(); // type (9)
+                EncodingRule.decodeRawNumber(bb); // skip map length part
+
+                // enumerate keys of map
+                while (true) {
+                        // parse map key
+                        String s = EncodingRule.decodeString(bb);
+                        if (s.equals("manifest_id")) {
+                                return EncodingRule.decodeInt(bb);
+                        } else {
+                                // parse map value
+                                bb.get();
+                                long l = EncodingRule.decodeRawNumber(bb);
+                                bb.position((int) (bb.position() + l));
+                        }
+                }
         }
 

kraken-confdb/src/main/java/org/krakenapps/confdb/file/FileConfigCollection.java

@@ -136 +136 @@
                 Manifest manifest = db.getManifest(changeset);
                 List<RevLog> snapshot = new ArrayList<RevLog>();
-
                 long count = reader.count();
                 for (long index = 0; index < count; index++) {
@@ -145 +144 @@
                                 snapshot.add(log);
                 }
-
                 return snapshot;
         }
@@ -177 +175 @@
                         // write collection log
                         int docId = writer.write(revlog);
-                        writer.sync();
 
                         // write db changelog

kraken-confdb/src/main/java/org/krakenapps/confdb/file/FileConfigDatabase.java

@@ -225 +225 @@
         public ConfigCollection getCollection(String name) {
                 try {
-                        Manifest manifest = getManifest(changeset);
+                        Manifest manifest = getManifest(changeset, true);
                         CollectionEntry col = manifest.getCollectionEntry(name);
                         if (col == null)
@@ -260 +260 @@
                 try {
                         Manifest manifest = null;
-                        if (xact == null)
-                                manifest = getManifest(changeset);
-                        else
+                        if (xact == null) {
+                                manifest = getManifest(changeset, true);
+                        } else
                                 manifest = xact.getManifest();
-
                         CollectionEntry col = manifest.getCollectionEntry(name);
 
@@ -331 +330 @@
         @Override
         public Manifest getManifest(Integer rev) {
+                return getManifest(rev, false);
+        }
+
+        public Manifest getManifest(Integer rev, boolean noConfigs) {
                 // read last changelog and get manifest doc id
                 int manifestId = 0;
@@ -345 +348 @@
                                 revlog = reader.findDoc(rev);
                         }
-
                         byte[] doc = reader.readDoc(revlog.getDocOffset(), revlog.getDocLength());
-                        ChangeLog change = ChangeLog.deserialize(doc);
-                        manifestId = change.getManifestId();
+                        manifestId = ChangeLog.getManifest(doc);
                 } catch (FileNotFoundException e) {
                         // changeset can be empty
@@ -366 +367 @@
                         RevLog revlog = reader.findDoc(manifestId);
                         byte[] doc = reader.readDoc(revlog.getDocOffset(), revlog.getDocLength());
-
                         // manifest id should be set here (id = revlog id)
-                        FileManifest manifest = FileManifest.deserialize(doc);
+                        FileManifest manifest = FileManifest.deserialize(doc, noConfigs);
                         manifest.setId(manifestId);
-
                         return manifest;
                 } catch (FileNotFoundException e) {

kraken-confdb/src/main/java/org/krakenapps/confdb/file/FileConfigIterator.java

@@ -118 +118 @@
                 byte[] b = reader.readDoc(log.getDocOffset(), log.getDocLength());
                 Object doc = EncodingRule.decode(ByteBuffer.wrap(b));
-
                 return new FileConfig(db, col, log.getDocId(), log.getRev(), log.getPrevRev(), doc, parser);
         }

kraken-confdb/src/main/java/org/krakenapps/confdb/file/FileManifest.java

@@ -23 +23 @@
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.TreeSet;
 
-import org.krakenapps.api.CollectionTypeHint;
 import org.krakenapps.api.FieldOption;
 import org.krakenapps.codec.EncodingRule;
@@ -35 +35 @@
         private int id;
 
-        @CollectionTypeHint(CollectionEntry.class)
+        @FieldOption(skip = true)
         private List<CollectionEntry> cols = new ArrayList<CollectionEntry>();
 
-        @CollectionTypeHint(ConfigEntry.class)
-        private List<ConfigEntry> configs = new ArrayList<ConfigEntry>();
-
-        /**
-         * just for existence test. we cannot change configs type (need ordering)
-         */
         @FieldOption(skip = true)
-        private Map<ConfigMatchKey, Long> tester = new HashMap<ConfigMatchKey, Long>();
+        private Map<ConfigMatchKey, ConfigEntry> configs = new TreeMap<ConfigMatchKey, ConfigEntry>();
 
         public FileManifest() {
@@ -73 +67 @@
         @Override
         public void add(ConfigEntry e) {
-                configs.remove(e); // remove duplicates
-                configs.add(e);
-                tester.put(new ConfigMatchKey(e), e.getRev());
+                configs.put(new ConfigMatchKey(e), e);
         }
 
         @Override
         public void remove(ConfigEntry e) {
-                configs.remove(e);
-                tester.remove(new ConfigMatchKey(e));
+                configs.remove(new ConfigMatchKey(e));
         }
 
@@ -107 +98 @@
 
                 List<ConfigEntry> entries = new LinkedList<ConfigEntry>();
-                for (ConfigEntry e : configs)
+                for (ConfigEntry e : configs.values())
                         if (e.getColId() == colId)
                                 entries.add(e);
@@ -125 +116 @@
                 CollectionEntry col = getCollectionEntry(colName);
                 ConfigMatchKey key = new ConfigMatchKey(col.getId(), docId);
-                Long r = tester.get(key);
-                return r != null && r == rev;
+                ConfigEntry e = configs.get(key);
+                return e != null && e.getRev() == rev;
         }
 
@@ -149 +140 @@
         private List<Object> serializeConfigs() {
                 List<Object> l = new ArrayList<Object>(configs.size());
-                for (ConfigEntry e : configs)
+                for (ConfigEntry e : configs.values())
                         l.add(new Object[] { e.getColId(), e.getDocId(), e.getRev() });
                 return l;
@@ -155 +146 @@
 
         public static FileManifest deserialize(byte[] b) {
+                return deserialize(b, false);
+        }
+
+        public static FileManifest deserialize(byte[] b, boolean noConfigs) {
                 ByteBuffer bb = ByteBuffer.wrap(b);
                 Map<String, Object> m = EncodingRule.decodeMap(bb);
@@ -172 +167 @@
                 }
 
+                // for ensureCollection() acceleration
+                if (noConfigs)
+                        return manifest;
+
                 for (Object o : (Object[]) m.get("configs")) {
                         if (o instanceof Map) {
@@ -195 +194 @@
         }
 
-        private static class ConfigMatchKey {
+        private static class ConfigMatchKey implements Comparable<ConfigMatchKey> {
                 private int colId;
                 private int docId;
@@ -206 +205 @@
                         this.colId = colId;
                         this.docId = docId;
+                }
+
+                @Override
+                public int compareTo(ConfigMatchKey o) {
+                        if (o == null)
+                                return -1;
+
+                        int value = colId - o.colId;
+                        if (value != 0)
+                                return value;
+
+                        value = docId - o.docId;
+                        if (value != 0)
+                                return value;
+
+                        return 0;
                 }