perf：升级Ip2Region
This commit is contained in:
曾文豪
@@ -3,12 +3,11 @@ package com.tiesheng.util.config;
|
||||
import cn.hutool.core.io.FileUtil;
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import cn.hutool.log.LogFactory;
|
||||
import com.tiesheng.util.ip2region.Searcher;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import javax.annotation.PostConstruct;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
/**
|
||||
* @author hao
|
||||
@@ -17,8 +16,8 @@ import java.io.RandomAccessFile;
|
||||
@ConfigurationProperties(prefix = "tiesheng.ip2region")
|
||||
public class Ip2regionConfig {
|
||||
|
||||
private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.db";
|
||||
private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.db";
|
||||
private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.xdb";
|
||||
private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.xdb";
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// 逻辑方法
|
||||
@@ -35,15 +34,20 @@ public class Ip2regionConfig {
|
||||
|
||||
|
||||
/**
|
||||
* 获取db文件
|
||||
* 搜索ip
|
||||
*
|
||||
* @param ip
|
||||
* @return
|
||||
*/
|
||||
public RandomAccessFile getDbAccessFile() throws FileNotFoundException {
|
||||
if (!FileUtil.exist(dbPath)) {
|
||||
downloadDbFile();
|
||||
public String search(String ip) {
|
||||
String region = "";
|
||||
try {
|
||||
Searcher searcher = Searcher.newWithFileOnly(dbPath);
|
||||
region = searcher.search(ip);
|
||||
searcher.close();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
return new RandomAccessFile(dbPath, "r");
|
||||
return region;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
/**
|
||||
* data block class
|
||||
*
|
||||
* @author chenxin<chenxin619315 @ gmail.com>
|
||||
*/
|
||||
public class DataBlock {
|
||||
/**
|
||||
* city id
|
||||
*/
|
||||
private int cityId;
|
||||
|
||||
/**
|
||||
* region address
|
||||
*/
|
||||
private String region;
|
||||
|
||||
/**
|
||||
* region ptr in the db file
|
||||
*/
|
||||
private int dataPtr;
|
||||
|
||||
/**
|
||||
* construct method
|
||||
*
|
||||
* @param cityId
|
||||
* @param region region string
|
||||
* @param dataPtr data ptr
|
||||
*/
|
||||
public DataBlock(int cityId, String region, int dataPtr) {
|
||||
this.cityId = cityId;
|
||||
this.region = region;
|
||||
this.dataPtr = dataPtr;
|
||||
}
|
||||
|
||||
public DataBlock(int cityId, String region) {
|
||||
this(cityId, region, 0);
|
||||
}
|
||||
|
||||
public int getCityId() {
|
||||
return cityId;
|
||||
}
|
||||
|
||||
public DataBlock setCityId(int cityId) {
|
||||
this.cityId = cityId;
|
||||
return this;
|
||||
}
|
||||
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public DataBlock setRegion(String region) {
|
||||
this.region = region;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getDataPtr() {
|
||||
return dataPtr;
|
||||
}
|
||||
|
||||
public DataBlock setDataPtr(int dataPtr) {
|
||||
this.dataPtr = dataPtr;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sb.append(cityId).append('|').append(region).append('|').append(dataPtr);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
/**
|
||||
* database configuration class
|
||||
*
|
||||
* @author chenxin<chenxin619315 @ gmail.com>
|
||||
*/
|
||||
public class DbConfig {
|
||||
/**
|
||||
* total header data block size
|
||||
*/
|
||||
private int totalHeaderSize;
|
||||
|
||||
/**
|
||||
* max index data block size
|
||||
* u should always choice the fastest read block size
|
||||
*/
|
||||
private int indexBlockSize;
|
||||
|
||||
/**
|
||||
* construct method
|
||||
*
|
||||
* @param totalHeaderSize
|
||||
*/
|
||||
public DbConfig(int totalHeaderSize) {
|
||||
this.totalHeaderSize = totalHeaderSize;
|
||||
this.indexBlockSize = 8192;
|
||||
}
|
||||
|
||||
public DbConfig() {
|
||||
this(8 * 2048);
|
||||
}
|
||||
|
||||
public int getTotalHeaderSize() {
|
||||
return totalHeaderSize;
|
||||
}
|
||||
|
||||
public DbConfig setTotalHeaderSize(int totalHeaderSize) {
|
||||
this.totalHeaderSize = totalHeaderSize;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getIndexBlockSize() {
|
||||
return indexBlockSize;
|
||||
}
|
||||
|
||||
public DbConfig setIndexBlockSize(int dataBlockSize) {
|
||||
this.indexBlockSize = dataBlockSize;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
/**
|
||||
* @author hao
|
||||
*/
|
||||
public class Header {
|
||||
public final int version;
|
||||
public final int indexPolicy;
|
||||
public final int createdAt;
|
||||
public final int startIndexPtr;
|
||||
public final int endIndexPtr;
|
||||
public final byte[] buffer;
|
||||
|
||||
public Header(byte[] buff) {
|
||||
assert buff.length >= 16;
|
||||
version = Searcher.getInt2(buff, 0);
|
||||
indexPolicy = Searcher.getInt2(buff, 2);
|
||||
createdAt = Searcher.getInt(buff, 4);
|
||||
startIndexPtr = Searcher.getInt(buff, 8);
|
||||
endIndexPtr = Searcher.getInt(buff, 12);
|
||||
buffer = buff;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "{" +
|
||||
"Version: " + version + ',' +
|
||||
"IndexPolicy: " + indexPolicy + ',' +
|
||||
"CreatedAt: " + createdAt + ',' +
|
||||
"StartIndexPtr: " + startIndexPtr + ',' +
|
||||
"EndIndexPtr: " + endIndexPtr +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
/**
|
||||
* item index class
|
||||
*
|
||||
* @author chenxin<chenxin619315 @ gmail.com>
|
||||
*/
|
||||
public class IndexBlock {
|
||||
|
||||
private static int LENGTH = 12;
|
||||
|
||||
/**
|
||||
* start ip address
|
||||
*/
|
||||
private long startIp;
|
||||
|
||||
/**
|
||||
* end ip address
|
||||
*/
|
||||
private long endIp;
|
||||
|
||||
/**
|
||||
* data ptr and data length
|
||||
*/
|
||||
private int dataPtr;
|
||||
|
||||
/**
|
||||
* data length
|
||||
*/
|
||||
private int dataLen;
|
||||
|
||||
public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) {
|
||||
this.startIp = startIp;
|
||||
this.endIp = endIp;
|
||||
this.dataPtr = dataPtr;
|
||||
this.dataLen = dataLen;
|
||||
}
|
||||
|
||||
public static int getIndexBlockLength() {
|
||||
return LENGTH;
|
||||
}
|
||||
|
||||
public long getStartIp() {
|
||||
return startIp;
|
||||
}
|
||||
|
||||
public IndexBlock setStartIp(long startIp) {
|
||||
this.startIp = startIp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public long getEndIp() {
|
||||
return endIp;
|
||||
}
|
||||
|
||||
public IndexBlock setEndIp(long endIp) {
|
||||
this.endIp = endIp;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getDataPtr() {
|
||||
return dataPtr;
|
||||
}
|
||||
|
||||
public IndexBlock setDataPtr(int dataPtr) {
|
||||
this.dataPtr = dataPtr;
|
||||
return this;
|
||||
}
|
||||
|
||||
public int getDataLen() {
|
||||
return dataLen;
|
||||
}
|
||||
|
||||
public IndexBlock setDataLen(int dataLen) {
|
||||
this.dataLen = dataLen;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the bytes for storage
|
||||
*
|
||||
* @return byte[]
|
||||
*/
|
||||
public byte[] getBytes() {
|
||||
/*
|
||||
* +------------+-----------+-----------+
|
||||
* | 4bytes | 4bytes | 4bytes |
|
||||
* +------------+-----------+-----------+
|
||||
* start ip end ip data ptr + len
|
||||
*/
|
||||
byte[] b = new byte[12];
|
||||
|
||||
IpUtil.writeIntLong(b, 0, startIp); //start ip
|
||||
IpUtil.writeIntLong(b, 4, endIp); //end ip
|
||||
|
||||
//write the data ptr and the length
|
||||
long mix = dataPtr | ((dataLen << 24) & 0xFF000000L);
|
||||
IpUtil.writeIntLong(b, 8, mix);
|
||||
|
||||
return b;
|
||||
}
|
||||
}
|
||||
@@ -1,416 +0,0 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
import cn.hutool.extra.spring.SpringUtil;
|
||||
import com.tiesheng.util.config.Ip2regionConfig;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/**
|
||||
* ip db searcher class (Not thread safe)
|
||||
*
|
||||
* @author chenxin<chenxin619315 @ gmail.com>
|
||||
*/
|
||||
public class Ip2Region {
|
||||
|
||||
|
||||
private static Ip2Region ip2Region;
|
||||
/**
|
||||
* db config
|
||||
*/
|
||||
private final DbConfig dbConfig;
|
||||
/**
|
||||
* db file access handler
|
||||
*/
|
||||
private RandomAccessFile raf = null;
|
||||
/**
|
||||
* header blocks buffer
|
||||
*/
|
||||
private long[] headerSip = null;
|
||||
private int[] headerPtr = null;
|
||||
private int headerLength;
|
||||
/**
|
||||
* super blocks info
|
||||
*/
|
||||
private long firstIndexPtr = 0;
|
||||
private long lastIndexPtr = 0;
|
||||
private int totalIndexBlocks = 0;
|
||||
/**
|
||||
* for memory mode
|
||||
* the original db binary string
|
||||
*/
|
||||
private byte[] dbBinStr = null;
|
||||
|
||||
/**
|
||||
* construct class
|
||||
*/
|
||||
public Ip2Region() {
|
||||
this.dbConfig = new DbConfig();
|
||||
try {
|
||||
Ip2regionConfig ip2regionConfig = SpringUtil.getBean(Ip2regionConfig.class);
|
||||
raf = ip2regionConfig.getDbAccessFile();
|
||||
} catch (Exception ignored) {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取单例
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public static Ip2Region getInstance() {
|
||||
if (ip2Region == null) {
|
||||
synchronized (Ip2Region.class) {
|
||||
if (ip2Region == null) {
|
||||
ip2Region = new Ip2Region();
|
||||
}
|
||||
}
|
||||
}
|
||||
return ip2Region;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region with an int ip address with memory binary search algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock memorySearch(long ip) throws IOException {
|
||||
int blen = IndexBlock.getIndexBlockLength();
|
||||
if (dbBinStr == null) {
|
||||
dbBinStr = new byte[(int) raf.length()];
|
||||
raf.seek(0L);
|
||||
raf.readFully(dbBinStr, 0, dbBinStr.length);
|
||||
|
||||
//initialize the global vars
|
||||
firstIndexPtr = IpUtil.getIntLong(dbBinStr, 0);
|
||||
lastIndexPtr = IpUtil.getIntLong(dbBinStr, 4);
|
||||
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
|
||||
}
|
||||
|
||||
//search the index blocks to define the data
|
||||
int l = 0, h = totalIndexBlocks;
|
||||
long sip, eip, dataptr = 0;
|
||||
while (l <= h) {
|
||||
int m = (l + h) >> 1;
|
||||
int p = (int) (firstIndexPtr + m * blen);
|
||||
|
||||
sip = IpUtil.getIntLong(dbBinStr, p);
|
||||
if (ip < sip) {
|
||||
h = m - 1;
|
||||
} else {
|
||||
eip = IpUtil.getIntLong(dbBinStr, p + 4);
|
||||
if (ip > eip) {
|
||||
l = m + 1;
|
||||
} else {
|
||||
dataptr = IpUtil.getIntLong(dbBinStr, p + 8);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//not matched
|
||||
if (dataptr == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//get the data
|
||||
int dataLen = (int) ((dataptr >> 24) & 0xFF);
|
||||
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
|
||||
int cityId = (int) IpUtil.getIntLong(dbBinStr, dataPtr);
|
||||
String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, StandardCharsets.UTF_8);
|
||||
|
||||
return new DataBlock(cityId, region, dataPtr);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region throught the ip address with memory binary search algorithm
|
||||
*
|
||||
* @return DataBlock
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock memorySearch() throws IOException {
|
||||
return memorySearch(null);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region throught the ip address with memory binary search algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @return DataBlock
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock memorySearch(String ip) throws IOException {
|
||||
return memorySearch(IpUtil.ip2long(ip));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* get by index ptr
|
||||
*
|
||||
* @param ptr
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock getByIndexPtr(long ptr) throws IOException {
|
||||
raf.seek(ptr);
|
||||
byte[] buffer = new byte[12];
|
||||
raf.readFully(buffer, 0, buffer.length);
|
||||
long extra = IpUtil.getIntLong(buffer, 8);
|
||||
|
||||
int dataLen = (int) ((extra >> 24) & 0xFF);
|
||||
int dataPtr = (int) ((extra & 0x00FFFFFF));
|
||||
|
||||
raf.seek(dataPtr);
|
||||
byte[] data = new byte[dataLen];
|
||||
raf.readFully(data, 0, data.length);
|
||||
|
||||
int cityId = (int) IpUtil.getIntLong(data, 0);
|
||||
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
|
||||
|
||||
return new DataBlock(cityId, region, dataPtr);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region with an int ip address with b-tree algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock btreeSearch(long ip) throws IOException {
|
||||
//check and load the header
|
||||
if (headerSip == null) {
|
||||
raf.seek(8L);
|
||||
byte[] b = new byte[dbConfig.getTotalHeaderSize()];
|
||||
raf.readFully(b, 0, b.length);
|
||||
|
||||
//fill the header
|
||||
int len = b.length >> 3, idx = 0;
|
||||
headerSip = new long[len];
|
||||
headerPtr = new int[len];
|
||||
long startIp, dataPtr;
|
||||
for (int i = 0; i < b.length; i += 8) {
|
||||
startIp = IpUtil.getIntLong(b, i);
|
||||
dataPtr = IpUtil.getIntLong(b, i + 4);
|
||||
if (dataPtr == 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
headerSip[idx] = startIp;
|
||||
headerPtr[idx] = (int) dataPtr;
|
||||
idx++;
|
||||
}
|
||||
|
||||
headerLength = idx;
|
||||
}
|
||||
|
||||
//1. define the index block with the binary search
|
||||
if (ip == headerSip[0]) {
|
||||
return getByIndexPtr(headerPtr[0]);
|
||||
} else if (ip == headerSip[headerLength - 1]) {
|
||||
return getByIndexPtr(headerPtr[headerLength - 1]);
|
||||
}
|
||||
|
||||
int l = 0, h = headerLength, sptr = 0, eptr = 0;
|
||||
while (l <= h) {
|
||||
int m = (l + h) >> 1;
|
||||
|
||||
//perfetc matched, just return it
|
||||
if (ip == headerSip[m]) {
|
||||
if (m > 0) {
|
||||
sptr = headerPtr[m - 1];
|
||||
eptr = headerPtr[m];
|
||||
} else {
|
||||
sptr = headerPtr[m];
|
||||
eptr = headerPtr[m + 1];
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
//less then the middle value
|
||||
if (ip < headerSip[m]) {
|
||||
if (m == 0) {
|
||||
sptr = headerPtr[m];
|
||||
eptr = headerPtr[m + 1];
|
||||
break;
|
||||
} else if (ip > headerSip[m - 1]) {
|
||||
sptr = headerPtr[m - 1];
|
||||
eptr = headerPtr[m];
|
||||
break;
|
||||
}
|
||||
h = m - 1;
|
||||
} else {
|
||||
if (m == headerLength - 1) {
|
||||
sptr = headerPtr[m - 1];
|
||||
eptr = headerPtr[m];
|
||||
break;
|
||||
} else if (ip <= headerSip[m + 1]) {
|
||||
sptr = headerPtr[m];
|
||||
eptr = headerPtr[m + 1];
|
||||
break;
|
||||
}
|
||||
l = m + 1;
|
||||
}
|
||||
}
|
||||
|
||||
//match nothing just stop it
|
||||
if (sptr == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//2. search the index blocks to define the data
|
||||
int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength();
|
||||
//include the right border block
|
||||
byte[] iBuffer = new byte[blockLen + blen];
|
||||
raf.seek(sptr);
|
||||
raf.readFully(iBuffer, 0, iBuffer.length);
|
||||
|
||||
l = 0;
|
||||
h = blockLen / blen;
|
||||
long sip, eip, dataptr = 0;
|
||||
while (l <= h) {
|
||||
int m = (l + h) >> 1;
|
||||
int p = m * blen;
|
||||
sip = IpUtil.getIntLong(iBuffer, p);
|
||||
if (ip < sip) {
|
||||
h = m - 1;
|
||||
} else {
|
||||
eip = IpUtil.getIntLong(iBuffer, p + 4);
|
||||
if (ip > eip) {
|
||||
l = m + 1;
|
||||
} else {
|
||||
dataptr = IpUtil.getIntLong(iBuffer, p + 8);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//not matched
|
||||
if (dataptr == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//3. get the data
|
||||
int dataLen = (int) ((dataptr >> 24) & 0xFF);
|
||||
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
|
||||
|
||||
raf.seek(dataPtr);
|
||||
byte[] data = new byte[dataLen];
|
||||
raf.readFully(data, 0, data.length);
|
||||
|
||||
int cityId = (int) IpUtil.getIntLong(data, 0);
|
||||
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
|
||||
|
||||
return new DataBlock(cityId, region, dataPtr);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region throught the ip address with b-tree search algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @return DataBlock
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock btreeSearch(String ip) {
|
||||
try {
|
||||
return btreeSearch(IpUtil.ip2long(ip));
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
return new DataBlock(0, "未知IP");
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region with a int ip address with binary search algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock binarySearch(long ip) throws IOException {
|
||||
int blen = IndexBlock.getIndexBlockLength();
|
||||
if (totalIndexBlocks == 0) {
|
||||
raf.seek(0L);
|
||||
byte[] superBytes = new byte[8];
|
||||
raf.readFully(superBytes, 0, superBytes.length);
|
||||
//initialize the global vars
|
||||
firstIndexPtr = IpUtil.getIntLong(superBytes, 0);
|
||||
lastIndexPtr = IpUtil.getIntLong(superBytes, 4);
|
||||
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
|
||||
}
|
||||
|
||||
//search the index blocks to define the data
|
||||
int l = 0, h = totalIndexBlocks;
|
||||
byte[] buffer = new byte[blen];
|
||||
long sip, eip, dataptr = 0;
|
||||
while (l <= h) {
|
||||
int m = (l + h) >> 1;
|
||||
raf.seek(firstIndexPtr + m * blen);
|
||||
raf.readFully(buffer, 0, buffer.length);
|
||||
sip = IpUtil.getIntLong(buffer, 0);
|
||||
if (ip < sip) {
|
||||
h = m - 1;
|
||||
} else {
|
||||
eip = IpUtil.getIntLong(buffer, 4);
|
||||
if (ip > eip) {
|
||||
l = m + 1;
|
||||
} else {
|
||||
dataptr = IpUtil.getIntLong(buffer, 8);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//not matched
|
||||
if (dataptr == 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
//get the data
|
||||
int dataLen = (int) ((dataptr >> 24) & 0xFF);
|
||||
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
|
||||
|
||||
raf.seek(dataPtr);
|
||||
byte[] data = new byte[dataLen];
|
||||
raf.readFully(data, 0, data.length);
|
||||
|
||||
int cityId = (int) IpUtil.getIntLong(data, 0);
|
||||
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
|
||||
|
||||
return new DataBlock(cityId, region, dataPtr);
|
||||
}
|
||||
|
||||
/**
|
||||
* get the region throught the ip address with binary search algorithm
|
||||
*
|
||||
* @param ip
|
||||
* @return DataBlock
|
||||
* @throws IOException
|
||||
*/
|
||||
public DataBlock binarySearch(String ip) throws IOException {
|
||||
return binarySearch(IpUtil.ip2long(ip));
|
||||
}
|
||||
|
||||
/**
|
||||
* get the db config
|
||||
*
|
||||
* @return DbConfig
|
||||
*/
|
||||
public DbConfig getDbConfig() {
|
||||
return dbConfig;
|
||||
}
|
||||
|
||||
/**
|
||||
* close the db
|
||||
*
|
||||
* @throws IOException
|
||||
*/
|
||||
public void close() throws IOException {
|
||||
headerSip = null;
|
||||
headerPtr = null;
|
||||
dbBinStr = null;
|
||||
if (raf != null) {
|
||||
raf.close();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
/**
|
||||
* util class
|
||||
*
|
||||
* @author chenxin<chenxin619315 @ gmail.com>
|
||||
*/
|
||||
public class IpUtil {
|
||||
/**
|
||||
* write specfield bytes to a byte array start from offset
|
||||
*
|
||||
* @param b
|
||||
* @param offset
|
||||
* @param v
|
||||
* @param bytes
|
||||
*/
|
||||
public static void write(byte[] b, int offset, long v, int bytes) {
|
||||
for (int i = 0; i < bytes; i++) {
|
||||
b[offset++] = (byte) ((v >>> (8 * i)) & 0xFF);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* write a int to a byte array
|
||||
*
|
||||
* @param b
|
||||
* @param offset
|
||||
* @param v
|
||||
*/
|
||||
public static void writeIntLong(byte[] b, int offset, long v) {
|
||||
b[offset++] = (byte) ((v >> 0) & 0xFF);
|
||||
b[offset++] = (byte) ((v >> 8) & 0xFF);
|
||||
b[offset++] = (byte) ((v >> 16) & 0xFF);
|
||||
b[offset] = (byte) ((v >> 24) & 0xFF);
|
||||
}
|
||||
|
||||
/**
|
||||
* get a int from a byte array start from the specifiled offset
|
||||
*
|
||||
* @param b
|
||||
* @param offset
|
||||
*/
|
||||
public static long getIntLong(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++] & 0x000000FFL)) |
|
||||
((b[offset++] << 8) & 0x0000FF00L) |
|
||||
((b[offset++] << 16) & 0x00FF0000L) |
|
||||
((b[offset] << 24) & 0xFF000000L)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* get a int from a byte array start from the specifield offset
|
||||
*
|
||||
* @param b
|
||||
* @param offset
|
||||
*/
|
||||
public static int getInt3(byte[] b, int offset) {
|
||||
return (
|
||||
(b[offset++] & 0x000000FF) |
|
||||
(b[offset++] & 0x0000FF00) |
|
||||
(b[offset] & 0x00FF0000)
|
||||
);
|
||||
}
|
||||
|
||||
public static int getInt2(byte[] b, int offset) {
|
||||
return (
|
||||
(b[offset++] & 0x000000FF) |
|
||||
(b[offset] & 0x0000FF00)
|
||||
);
|
||||
}
|
||||
|
||||
public static int getInt1(byte[] b, int offset) {
|
||||
return (
|
||||
(b[offset] & 0x000000FF)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* string ip to long ip
|
||||
*
|
||||
* @param ip
|
||||
* @return long
|
||||
*/
|
||||
public static long ip2long(String ip) {
|
||||
String[] p = ip.split("\\.");
|
||||
if (p.length != 4) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int p1 = ((Integer.valueOf(p[0]) << 24) & 0xFF000000);
|
||||
int p2 = ((Integer.valueOf(p[1]) << 16) & 0x00FF0000);
|
||||
int p3 = ((Integer.valueOf(p[2]) << 8) & 0x0000FF00);
|
||||
int p4 = ((Integer.valueOf(p[3]) << 0) & 0x000000FF);
|
||||
|
||||
return ((p1 | p2 | p3 | p4) & 0xFFFFFFFFL);
|
||||
}
|
||||
|
||||
/**
|
||||
* int to ip string
|
||||
*
|
||||
* @param ip
|
||||
* @return string
|
||||
*/
|
||||
public static String long2ip(long ip) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
|
||||
sb
|
||||
.append((ip >> 24) & 0xFF).append('.')
|
||||
.append((ip >> 16) & 0xFF).append('.')
|
||||
.append((ip >> 8) & 0xFF).append('.')
|
||||
.append((ip >> 0) & 0xFF);
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
package com.tiesheng.util.ip2region;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
|
||||
public class Searcher {
|
||||
// constant defined copied from the xdb maker
|
||||
public static final int HeaderInfoLength = 256;
|
||||
public static final int VectorIndexRows = 256;
|
||||
public static final int VectorIndexCols = 256;
|
||||
public static final int VectorIndexSize = 8;
|
||||
public static final int SegmentIndexSize = 14;
|
||||
public static final byte[] shiftIndex = {24, 16, 8, 0};
|
||||
// random access file handle for file based search
|
||||
private final RandomAccessFile handle;
|
||||
// vector index.
|
||||
// use the byte[] instead of VectorIndex entry array to keep
|
||||
// the minimal memory allocation.
|
||||
private final byte[] vectorIndex;
|
||||
|
||||
// xdb content buffer, used for in-memory search
|
||||
private final byte[] contentBuff;
|
||||
|
||||
// --- static method to create searchers
|
||||
private int ioCount = 0;
|
||||
|
||||
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
|
||||
if (cBuff != null) {
|
||||
this.handle = null;
|
||||
this.vectorIndex = null;
|
||||
this.contentBuff = cBuff;
|
||||
} else {
|
||||
this.handle = new RandomAccessFile(dbFile, "r");
|
||||
this.vectorIndex = vectorIndex;
|
||||
this.contentBuff = null;
|
||||
}
|
||||
}
|
||||
|
||||
public static Searcher newWithFileOnly(String dbPath) throws IOException {
|
||||
return new Searcher(dbPath, null, null);
|
||||
}
|
||||
|
||||
// --- End of creator
|
||||
|
||||
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
|
||||
return new Searcher(dbPath, vectorIndex, null);
|
||||
}
|
||||
|
||||
public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
|
||||
return new Searcher(null, null, cBuff);
|
||||
}
|
||||
|
||||
public static Header loadHeader(RandomAccessFile handle) throws IOException {
|
||||
handle.seek(0);
|
||||
final byte[] buff = new byte[HeaderInfoLength];
|
||||
handle.read(buff);
|
||||
return new Header(buff);
|
||||
}
|
||||
|
||||
public static Header loadHeaderFromFile(String dbPath) throws IOException {
|
||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||
final Header header = loadHeader(handle);
|
||||
handle.close();
|
||||
return header;
|
||||
}
|
||||
|
||||
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
|
||||
handle.seek(HeaderInfoLength);
|
||||
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
|
||||
final byte[] buff = new byte[len];
|
||||
int rLen = handle.read(buff);
|
||||
if (rLen != len) {
|
||||
throw new IOException("incomplete read: read bytes should be " + len);
|
||||
}
|
||||
|
||||
return buff;
|
||||
}
|
||||
|
||||
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
|
||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||
final byte[] vIndex = loadVectorIndex(handle);
|
||||
handle.close();
|
||||
return vIndex;
|
||||
}
|
||||
|
||||
// --- static cache util function
|
||||
|
||||
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
|
||||
handle.seek(0);
|
||||
final byte[] buff = new byte[(int) handle.length()];
|
||||
int rLen = handle.read(buff);
|
||||
if (rLen != buff.length) {
|
||||
throw new IOException("incomplete read: read bytes should be " + buff.length);
|
||||
}
|
||||
|
||||
return buff;
|
||||
}
|
||||
|
||||
public static byte[] loadContentFromFile(String dbPath) throws IOException {
|
||||
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
|
||||
final byte[] content = loadContent(handle);
|
||||
handle.close();
|
||||
return content;
|
||||
}
|
||||
|
||||
/* get an int from a byte array start from the specified offset */
|
||||
public static long getIntLong(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++] & 0x000000FFL)) |
|
||||
((b[offset++] << 8) & 0x0000FF00L) |
|
||||
((b[offset++] << 16) & 0x00FF0000L) |
|
||||
((b[offset] << 24) & 0xFF000000L)
|
||||
);
|
||||
}
|
||||
|
||||
public static int getInt(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++] & 0x000000FF)) |
|
||||
((b[offset++] << 8) & 0x0000FF00) |
|
||||
((b[offset++] << 16) & 0x00FF0000) |
|
||||
((b[offset] << 24) & 0xFF000000)
|
||||
);
|
||||
}
|
||||
|
||||
public static int getInt2(byte[] b, int offset) {
|
||||
return (
|
||||
((b[offset++] & 0x000000FF)) |
|
||||
((b[offset] << 8) & 0x0000FF00)
|
||||
);
|
||||
}
|
||||
|
||||
/* long int to ip string */
|
||||
public static String long2ip(long ip) {
|
||||
return String.valueOf((ip >> 24) & 0xFF) + '.' +
|
||||
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
|
||||
}
|
||||
|
||||
// --- End cache load util function
|
||||
|
||||
// --- static util method
|
||||
|
||||
/* check the specified ip address */
|
||||
public static long checkIP(String ip) throws Exception {
|
||||
String[] ps = ip.split("\\.");
|
||||
if (ps.length != 4) {
|
||||
throw new Exception("invalid ip address `" + ip + "`");
|
||||
}
|
||||
|
||||
long ipDst = 0;
|
||||
for (int i = 0; i < ps.length; i++) {
|
||||
int val = Integer.parseInt(ps[i]);
|
||||
if (val > 255) {
|
||||
throw new Exception("ip part `" + ps[i] + "` should be less then 256");
|
||||
}
|
||||
|
||||
ipDst |= ((long) val << shiftIndex[i]);
|
||||
}
|
||||
|
||||
return ipDst & 0xFFFFFFFFL;
|
||||
}
|
||||
|
||||
public void close() throws IOException {
|
||||
if (this.handle != null) {
|
||||
this.handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
public int getIOCount() {
|
||||
return ioCount;
|
||||
}
|
||||
|
||||
public String search(String ipStr) throws Exception {
|
||||
long ip = checkIP(ipStr);
|
||||
return search(ip);
|
||||
}
|
||||
|
||||
public String search(long ip) throws IOException {
|
||||
// reset the global counter
|
||||
this.ioCount = 0;
|
||||
|
||||
// locate the segment index block based on the vector index
|
||||
int sPtr = 0, ePtr = 0;
|
||||
int il0 = (int) ((ip >> 24) & 0xFF);
|
||||
int il1 = (int) ((ip >> 16) & 0xFF);
|
||||
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
|
||||
// System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
|
||||
if (vectorIndex != null) {
|
||||
sPtr = getInt(vectorIndex, idx);
|
||||
ePtr = getInt(vectorIndex, idx + 4);
|
||||
} else if (contentBuff != null) {
|
||||
sPtr = getInt(contentBuff, HeaderInfoLength + idx);
|
||||
ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4);
|
||||
} else {
|
||||
final byte[] buff = new byte[VectorIndexSize];
|
||||
read(HeaderInfoLength + idx, buff);
|
||||
sPtr = getInt(buff, 0);
|
||||
ePtr = getInt(buff, 4);
|
||||
}
|
||||
|
||||
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
|
||||
|
||||
// binary search the segment index block to get the region info
|
||||
final byte[] buff = new byte[SegmentIndexSize];
|
||||
int dataLen = -1, dataPtr = -1;
|
||||
int l = 0, h = (ePtr - sPtr) / SegmentIndexSize;
|
||||
while (l <= h) {
|
||||
int m = (l + h) >> 1;
|
||||
int p = sPtr + m * SegmentIndexSize;
|
||||
|
||||
// read the segment index
|
||||
read(p, buff);
|
||||
long sip = getIntLong(buff, 0);
|
||||
if (ip < sip) {
|
||||
h = m - 1;
|
||||
} else {
|
||||
long eip = getIntLong(buff, 4);
|
||||
if (ip > eip) {
|
||||
l = m + 1;
|
||||
} else {
|
||||
dataLen = getInt2(buff, 8);
|
||||
dataPtr = getInt(buff, 10);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// empty match interception
|
||||
// System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
|
||||
if (dataPtr < 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// load and return the region data
|
||||
final byte[] regionBuff = new byte[dataLen];
|
||||
read(dataPtr, regionBuff);
|
||||
return new String(regionBuff, "utf-8");
|
||||
}
|
||||
|
||||
protected void read(int offset, byte[] buffer) throws IOException {
|
||||
// check the in-memory buffer first
|
||||
if (contentBuff != null) {
|
||||
// @TODO: reduce data copying, directly decode the data ?
|
||||
System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
|
||||
return;
|
||||
}
|
||||
|
||||
// read from the file handle
|
||||
assert handle != null;
|
||||
handle.seek(offset);
|
||||
|
||||
this.ioCount++;
|
||||
int rLen = handle.read(buffer);
|
||||
if (rLen != buffer.length) {
|
||||
throw new IOException("incomplete read: read bytes should be " + buffer.length);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user