Update jleveldb
This commit is contained in:
Victor Shcherb
parent
e2f30785a6
commit
dc1d6ce7c9
6 changed files with 14 additions and 217 deletions
|
|
@ -8,7 +8,7 @@
|
|||
<classpathentry kind="lib" path="lib/json-20090211.jar"/>
|
||||
<classpathentry kind="lib" path="lib/h2-latest.jar"/>
|
||||
<classpathentry kind="lib" path="lib/bsh-core-2.0b4.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jleveldb-v01.jar"/>
|
||||
<classpathentry kind="lib" path="lib/junidecode-0.1.jar"/>
|
||||
<classpathentry kind="lib" path="lib/jleveldb-v01.jar"/>
|
||||
<classpathentry kind="output" path="bin"/>
|
||||
</classpath>
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -78,8 +78,8 @@ public enum DBDialect {
|
|||
if (DBDialect.NOSQL == this) {
|
||||
DBAccessor dbAccessor = LevelDBAccess.getDBAcessor();
|
||||
Options opts = new Options();
|
||||
opts.setCreate_if_missing(true);
|
||||
Status status = dbAccessor.Open(opts, fileName);
|
||||
opts.setCreateIfMissing(true);
|
||||
Status status = dbAccessor.open(opts, fileName);
|
||||
if(!status.ok()){
|
||||
throw new SQLException(status.ToString());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,12 +10,10 @@ import java.util.LinkedHashMap;
|
|||
import java.util.Map;
|
||||
|
||||
import net.osmand.IProgress;
|
||||
import net.osmand.osm.ArraySerializer;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.Node;
|
||||
import net.osmand.osm.Relation;
|
||||
import net.osmand.osm.Way;
|
||||
import net.osmand.osm.ArraySerializer.EntityValueTokenizer;
|
||||
import net.osmand.osm.Entity.EntityId;
|
||||
import net.osmand.osm.Entity.EntityType;
|
||||
|
||||
|
|
@ -23,6 +21,8 @@ import org.apache.commons.logging.Log;
|
|||
import org.apache.commons.logging.LogFactory;
|
||||
import org.json.JSONException;
|
||||
|
||||
import com.anvisics.jleveldb.ArraySerializer;
|
||||
import com.anvisics.jleveldb.ArraySerializer.EntityValueTokenizer;
|
||||
import com.anvisics.jleveldb.ext.DBAccessor;
|
||||
import com.anvisics.jleveldb.ext.DBIterator;
|
||||
import com.anvisics.jleveldb.ext.ReadOptions;
|
||||
|
|
@ -288,7 +288,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
for (EntityId i : ids) {
|
||||
char pr = i.getType() == EntityType.NODE ? '0' : (i.getType() == EntityType.WAY ? '1' : '2');
|
||||
String key = pr + "" + i.getId();
|
||||
String value = accessor.Get(randomAccessOptions, key);
|
||||
String value = accessor.get(randomAccessOptions, key);
|
||||
if (value != null && value.length() > 0) {
|
||||
try {
|
||||
Entity es = loadEntityNoSqlFromValue(randomAccessOptions, key, value, loadTags, false);
|
||||
|
|
@ -356,7 +356,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
int n = tokenizer.next();
|
||||
while(n == ArraySerializer.ELEMENT){
|
||||
String pointId = "0"+ tokenizer.value();
|
||||
String pointVal = this.accessor.Get(opts, pointId);
|
||||
String pointVal = this.accessor.get(opts, pointId);
|
||||
Node node = (Node) loadEntityNoSqlFromValue(opts, pointId, pointVal, false, false);
|
||||
if(node != null){
|
||||
((Way) e).addNode(node);
|
||||
|
|
@ -389,7 +389,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
|
||||
private int iterateOverEntitiesNoSQL(IProgress progress, EntityType type, OsmDbVisitor visitor) throws SQLException {
|
||||
ReadOptions opts = new ReadOptions();
|
||||
DBIterator iterator = accessor.NewIterator(opts);
|
||||
DBIterator iterator = accessor.newIterator(opts);
|
||||
String prefix = "0";
|
||||
int count = 0;
|
||||
if (type == EntityType.WAY) {
|
||||
|
|
@ -398,9 +398,9 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
prefix = "2";
|
||||
}
|
||||
|
||||
iterator.Seek(prefix);
|
||||
iterator.seek(prefix);
|
||||
|
||||
while(iterator.Valid()){
|
||||
while(iterator.valid()){
|
||||
String key = iterator.key();
|
||||
if(!key.startsWith(prefix)){
|
||||
break;
|
||||
|
|
@ -426,7 +426,7 @@ public class OsmDbAccessor implements OsmDbAccessorContext {
|
|||
} catch (JSONException e) {
|
||||
log.warn(key + " - " + e.getMessage() + " " + value + "("+value.length()+"]", e);
|
||||
}
|
||||
iterator.Next();
|
||||
iterator.next();
|
||||
}
|
||||
iterator.delete();
|
||||
return count;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import java.sql.Statement;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import net.osmand.osm.ArraySerializer;
|
||||
import net.osmand.osm.Entity;
|
||||
import net.osmand.osm.Node;
|
||||
import net.osmand.osm.Relation;
|
||||
|
|
@ -20,6 +19,7 @@ import net.osmand.osm.io.OsmBaseStorage;
|
|||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
import com.anvisics.jleveldb.ArraySerializer;
|
||||
import com.anvisics.jleveldb.ext.DBAccessor;
|
||||
import com.anvisics.jleveldb.ext.DBWriteBatch;
|
||||
import com.anvisics.jleveldb.ext.WriteOptions;
|
||||
|
|
@ -110,7 +110,7 @@ public class OsmDbCreator implements IOsmStorageFilter {
|
|||
}
|
||||
prepTags.close();
|
||||
} else {
|
||||
database.Write(options, batch);
|
||||
database.write(options, batch);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -179,7 +179,7 @@ public class OsmDbCreator implements IOsmStorageFilter {
|
|||
}
|
||||
batch.Put(key, serializeEntityWOId(e));
|
||||
if (currentCountNode > BATCH_SIZE_OSM) {
|
||||
database.Write(options, batch);
|
||||
database.write(options, batch);
|
||||
batch = new DBWriteBatch();
|
||||
long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
System.out.println("" + Runtime.getRuntime().totalMemory() / (1024 * 1024) + " MB Total " + (usedMemory / (1024 * 1024))
|
||||
|
|
|
|||
|
|
@ -1,203 +0,0 @@
|
|||
package net.osmand.osm;
|
||||
|
||||
/**
|
||||
* Serializes custom arrays of strings into string using [,] notation with quotation character \
|
||||
* Examples : [1, 2, 3, [4, 5]]
|
||||
* @author victor
|
||||
*
|
||||
*/
|
||||
public class ArraySerializer {
|
||||
|
||||
public static final int START_ARRAY = 1;
|
||||
public static final int ELEMENT = 2;
|
||||
public static final int END_ARRAY = 3;
|
||||
public static final int END = 4;
|
||||
|
||||
|
||||
public static class EntityValueTokenizer {
|
||||
private String tokenize;
|
||||
private int lastReadIndex;
|
||||
private String value;
|
||||
private boolean firstElementAfterArrayOpened;
|
||||
|
||||
public EntityValueTokenizer(){
|
||||
tokenize("");
|
||||
}
|
||||
public void tokenize(String v){
|
||||
lastReadIndex = 0;
|
||||
tokenize = v;
|
||||
value = "";
|
||||
firstElementAfterArrayOpened = false;
|
||||
}
|
||||
|
||||
public int next(){
|
||||
value ="";
|
||||
int currentInd = lastReadIndex;
|
||||
lastReadIndex ++;
|
||||
if(currentInd >= tokenize.length()){
|
||||
return END;
|
||||
}
|
||||
// check if current element is opening array
|
||||
if(tokenize.charAt(currentInd) == '['){
|
||||
firstElementAfterArrayOpened = true;
|
||||
return START_ARRAY;
|
||||
}
|
||||
// check if current element is closing array
|
||||
if(tokenize.charAt(currentInd) == ']'){
|
||||
return END_ARRAY;
|
||||
}
|
||||
|
||||
// check if current element is comma
|
||||
// 2 ways : skip it and treat as empty element
|
||||
int startRead = currentInd;
|
||||
if(tokenize.charAt(currentInd) == ','){
|
||||
if(firstElementAfterArrayOpened) {
|
||||
// special case for : '[,' return empty element
|
||||
firstElementAfterArrayOpened = false;
|
||||
lastReadIndex --;
|
||||
return ELEMENT;
|
||||
} else {
|
||||
startRead++;
|
||||
}
|
||||
}
|
||||
firstElementAfterArrayOpened = false;
|
||||
|
||||
// read characters till stop (, ] [ ) or end
|
||||
boolean previousSpecial = tokenize.charAt(currentInd) == '\\';
|
||||
// check quotation element \
|
||||
// elements to quote : , [ ] and \ (itself)
|
||||
while (++currentInd < tokenize.length()) {
|
||||
char c = tokenize.charAt(currentInd);
|
||||
if(c == '\\'){
|
||||
if(previousSpecial){
|
||||
value += tokenize.substring(startRead, currentInd - 1);
|
||||
startRead = currentInd;
|
||||
previousSpecial = false;
|
||||
} else {
|
||||
previousSpecial = true;
|
||||
}
|
||||
} else {
|
||||
if (c == ',' || c == ']' || c == '[') {
|
||||
if (!previousSpecial) {
|
||||
if (c == '[' && value.length() == 0) {
|
||||
lastReadIndex ++;
|
||||
firstElementAfterArrayOpened = true;
|
||||
return START_ARRAY;
|
||||
}
|
||||
break;
|
||||
} else {
|
||||
value += tokenize.substring(startRead, currentInd - 1);
|
||||
startRead = currentInd;
|
||||
}
|
||||
}
|
||||
previousSpecial = false;
|
||||
}
|
||||
lastReadIndex ++;
|
||||
}
|
||||
value += tokenize.substring(startRead, currentInd);
|
||||
return ELEMENT;
|
||||
}
|
||||
|
||||
|
||||
public String value(){
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
public static void startArray(StringBuilder builder, boolean first){
|
||||
if(!first){
|
||||
builder.append(",");
|
||||
}
|
||||
builder.append("[");
|
||||
}
|
||||
|
||||
public static void value(StringBuilder builder, String value, boolean first){
|
||||
if(!first){
|
||||
builder.append(",");
|
||||
}
|
||||
value = value.replace("\\", "\\\\");
|
||||
value = value.replace("[", "\\[").replace("]", "\\]").replace(",", "\\,");
|
||||
builder.append(value);
|
||||
}
|
||||
|
||||
public static void endArray(StringBuilder builder){
|
||||
builder.append("]");
|
||||
}
|
||||
|
||||
private static void testCase(String value, String[] comments){
|
||||
int next;
|
||||
EntityValueTokenizer tokenizer = new EntityValueTokenizer();
|
||||
tokenizer.tokenize(value);
|
||||
int currentInd = 0;
|
||||
while ((next = tokenizer.next()) != END) {
|
||||
if (next == ELEMENT) {
|
||||
assertEquals(comments[currentInd], tokenizer.value(), value + " " + currentInd);
|
||||
} else if (next == START_ARRAY) {
|
||||
assertEquals(comments[currentInd], "[", value + " " + currentInd);
|
||||
} else if (next == END_ARRAY) {
|
||||
assertEquals(comments[currentInd], "]", value+ " " + currentInd);
|
||||
}
|
||||
currentInd++;
|
||||
}
|
||||
|
||||
// serialization
|
||||
StringBuilder builder = new StringBuilder();
|
||||
boolean first = true;
|
||||
for (String st : comments) {
|
||||
if (st.equals("[")) {
|
||||
startArray(builder, first);
|
||||
first = true;
|
||||
} else if (st.equals("]")) {
|
||||
endArray(builder);
|
||||
} else {
|
||||
value(builder, st, first);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
String res = builder.toString().replace("\\\\", "\\");
|
||||
if(!res.equals(value) && !builder.toString().equals(value)){
|
||||
System.err.println("Serialization of value is probably wrong : " + res + " original " + value);
|
||||
}
|
||||
}
|
||||
|
||||
private static void assertEquals(String exp, String actual, String msg) {
|
||||
if (!exp.equals(actual)) {
|
||||
throw new IllegalArgumentException("Expected '" + exp + "' != '" + actual + "' : " + msg);
|
||||
}
|
||||
}
|
||||
|
||||
private static void testCases(){
|
||||
testCase("[1,2,[]]", new String[] {"[", "1", "2", "[", "]", "]"});
|
||||
testCase("[[1,2],[1,2]]", new String[] {"[", "[","1", "2", "]","[","1", "2", "]", "]"});
|
||||
|
||||
// quotation
|
||||
testCase("[1, 2, 4,[1,3\\,4]]", new String[] {"[", "1", " 2", " 4", "[", "1", "3,4","]", "]"});
|
||||
testCase("[1,4,\\[1,3\\,4\\]]", new String[] {"[", "1", "4", "[1", "3,4]", "]"});
|
||||
testCase("[1,\\4,3\\[,4\\]]", new String[] {"[", "1", "\\4", "3[", "4]", "]"});
|
||||
testCase("[1,\\4,3\\[,4\\]]", new String[] {"[", "1", "\\4", "3[", "4]", "]"});
|
||||
testCase("[1,\\\\,3]", new String[] {"[", "1", "\\", "3", "]"});
|
||||
// testCase("[1,\\\\,\\[]", new String[] {"[", "1", "\\", "[", "]"});
|
||||
testCase("[\\\\1,\\\\2,333\\\\]", new String[] {"[", "\\1", "\\2", "333\\", "]"});
|
||||
|
||||
testCase("[2,]", new String[] {"[", "2", "", "]"});
|
||||
testCase("[,2]", new String[] {"[", "", "2", "]"});
|
||||
testCase("[,2,,]", new String[] {"[", "", "2", "", "", "]"});
|
||||
testCase("[,,]", new String[] {"[", "", "", "", "]"});
|
||||
testCase("[1,,[,,]]", new String[] {"[", "1", "", "[", "", "", "", "]", "]"});
|
||||
|
||||
testCase("14555", new String[] {"14555"});
|
||||
testCase("\\[1\\]", new String[] {"[1]"});
|
||||
testCase("\\[1\\,2", new String[] {"[1,2"});
|
||||
System.out.println("All is successfull");
|
||||
|
||||
// tokenizer.tokenize("[1,,[,,]]"); // 1, '', ['','','']
|
||||
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
testCases();
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
Loading…
Reference in a new issue