diff --git a/springboot-ademo/src/main/java/com/tiesheng/demo/controller/TestController.java b/springboot-ademo/src/main/java/com/tiesheng/demo/controller/TestController.java index 0476453..cc01b44 100644 --- a/springboot-ademo/src/main/java/com/tiesheng/demo/controller/TestController.java +++ b/springboot-ademo/src/main/java/com/tiesheng/demo/controller/TestController.java @@ -8,6 +8,7 @@ import com.tiesheng.login.config.token.bean.TokenBean; import com.tiesheng.message.config.aliyun.AliyunSmsConfig; import com.tiesheng.message.pojos.MessageReqResp; import com.tiesheng.util.config.GlobalConfig; +import com.tiesheng.util.config.Ip2regionConfig; import com.tiesheng.util.pojos.ApiResp; import com.tiesheng.util.pojos.FileUploadPath; import org.springframework.beans.factory.annotation.Autowired; @@ -29,6 +30,9 @@ public class TestController { AliyunSmsConfig aliyunSmsConfig; @Autowired TsTokenConfig tsTokenConfig; + @Autowired + Ip2regionConfig ip2regionConfig; + @RequestMapping("/index") @TokenIgnore @@ -80,4 +84,12 @@ public class TestController { return ApiResp.respOK(uploadPath.getAbsolutePath()); } + + @RequestMapping("searchIP") + @TokenIgnore + public ApiResp searchIp() { + String search = ip2regionConfig.search("127.0.0.1"); + return ApiResp.respOK(search); + } + } diff --git a/springboot-util/src/main/java/com/tiesheng/util/config/Ip2regionConfig.java b/springboot-util/src/main/java/com/tiesheng/util/config/Ip2regionConfig.java index eea646d..33e97f1 100644 --- a/springboot-util/src/main/java/com/tiesheng/util/config/Ip2regionConfig.java +++ b/springboot-util/src/main/java/com/tiesheng/util/config/Ip2regionConfig.java @@ -3,12 +3,11 @@ package com.tiesheng.util.config; import cn.hutool.core.io.FileUtil; import cn.hutool.http.HttpUtil; import cn.hutool.log.LogFactory; +import com.tiesheng.util.ip2region.Searcher; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Configuration; import javax.annotation.PostConstruct; -import java.io.FileNotFoundException; -import java.io.RandomAccessFile; /** * @author hao @@ -17,8 +16,8 @@ import java.io.RandomAccessFile; @ConfigurationProperties(prefix = "tiesheng.ip2region") public class Ip2regionConfig { - private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.db"; - private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.db"; + private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.xdb"; + private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.xdb"; /////////////////////////////////////////////////////////////////////////// // 逻辑方法 @@ -35,15 +34,20 @@ public class Ip2regionConfig { /** - * 获取db文件 + * 搜索ip * + * @param ip * @return */ - public RandomAccessFile getDbAccessFile() throws FileNotFoundException { - if (!FileUtil.exist(dbPath)) { - downloadDbFile(); + public String search(String ip) { + String region = ""; + try { + Searcher searcher = Searcher.newWithFileOnly(dbPath); + region = searcher.search(ip); + searcher.close(); + } catch (Exception ignored) { } - return new RandomAccessFile(dbPath, "r"); + return region; } /////////////////////////////////////////////////////////////////////////// diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/DataBlock.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/DataBlock.java deleted file mode 100644 index 2ff402e..0000000 --- a/springboot-util/src/main/java/com/tiesheng/util/ip2region/DataBlock.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.tiesheng.util.ip2region; - -/** - * data block class - * - * @author chenxin - */ -public class DataBlock { - /** - * city id - */ - private int cityId; - - /** - * region address - */ - private String region; - - /** - * region ptr in the db file - */ - private int dataPtr; - - /** - * construct method - * - * @param cityId - * @param region region string - * @param dataPtr data ptr - */ - public DataBlock(int cityId, String region, int dataPtr) { - this.cityId = cityId; - this.region = region; - this.dataPtr = dataPtr; - } - - public DataBlock(int cityId, String region) { - this(cityId, region, 0); - } - - public int getCityId() { - return cityId; - } - - public DataBlock setCityId(int cityId) { - this.cityId = cityId; - return this; - } - - public String getRegion() { - return region; - } - - public DataBlock setRegion(String region) { - this.region = region; - return this; - } - - public int getDataPtr() { - return dataPtr; - } - - public DataBlock setDataPtr(int dataPtr) { - this.dataPtr = dataPtr; - return this; - } - - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(cityId).append('|').append(region).append('|').append(dataPtr); - return sb.toString(); - } - -} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/DbConfig.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/DbConfig.java deleted file mode 100644 index 4b333fc..0000000 --- a/springboot-util/src/main/java/com/tiesheng/util/ip2region/DbConfig.java +++ /dev/null @@ -1,51 +0,0 @@ -package com.tiesheng.util.ip2region; - -/** - * database configuration class - * - * @author chenxin - */ -public class DbConfig { - /** - * total header data block size - */ - private int totalHeaderSize; - - /** - * max index data block size - * u should always choice the fastest read block size - */ - private int indexBlockSize; - - /** - * construct method - * - * @param totalHeaderSize - */ - public DbConfig(int totalHeaderSize) { - this.totalHeaderSize = totalHeaderSize; - this.indexBlockSize = 8192; - } - - public DbConfig() { - this(8 * 2048); - } - - public int getTotalHeaderSize() { - return totalHeaderSize; - } - - public DbConfig setTotalHeaderSize(int totalHeaderSize) { - this.totalHeaderSize = totalHeaderSize; - return this; - } - - public int getIndexBlockSize() { - return indexBlockSize; - } - - public DbConfig setIndexBlockSize(int dataBlockSize) { - this.indexBlockSize = dataBlockSize; - return this; - } -} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/Header.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/Header.java new file mode 100644 index 0000000..aa785c9 --- /dev/null +++ b/springboot-util/src/main/java/com/tiesheng/util/ip2region/Header.java @@ -0,0 +1,34 @@ +package com.tiesheng.util.ip2region; + +/** + * @author hao + */ +public class Header { + public final int version; + public final int indexPolicy; + public final int createdAt; + public final int startIndexPtr; + public final int endIndexPtr; + public final byte[] buffer; + + public Header(byte[] buff) { + assert buff.length >= 16; + version = Searcher.getInt2(buff, 0); + indexPolicy = Searcher.getInt2(buff, 2); + createdAt = Searcher.getInt(buff, 4); + startIndexPtr = Searcher.getInt(buff, 8); + endIndexPtr = Searcher.getInt(buff, 12); + buffer = buff; + } + + @Override + public String toString() { + return "{" + + "Version: " + version + ',' + + "IndexPolicy: " + indexPolicy + ',' + + "CreatedAt: " + createdAt + ',' + + "StartIndexPtr: " + startIndexPtr + ',' + + "EndIndexPtr: " + endIndexPtr + + '}'; + } +} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/IndexBlock.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/IndexBlock.java deleted file mode 100644 index b31f130..0000000 --- a/springboot-util/src/main/java/com/tiesheng/util/ip2region/IndexBlock.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.tiesheng.util.ip2region; - -/** - * item index class - * - * @author chenxin - */ -public class IndexBlock { - - private static int LENGTH = 12; - - /** - * start ip address - */ - private long startIp; - - /** - * end ip address - */ - private long endIp; - - /** - * data ptr and data length - */ - private int dataPtr; - - /** - * data length - */ - private int dataLen; - - public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) { - this.startIp = startIp; - this.endIp = endIp; - this.dataPtr = dataPtr; - this.dataLen = dataLen; - } - - public static int getIndexBlockLength() { - return LENGTH; - } - - public long getStartIp() { - return startIp; - } - - public IndexBlock setStartIp(long startIp) { - this.startIp = startIp; - return this; - } - - public long getEndIp() { - return endIp; - } - - public IndexBlock setEndIp(long endIp) { - this.endIp = endIp; - return this; - } - - public int getDataPtr() { - return dataPtr; - } - - public IndexBlock setDataPtr(int dataPtr) { - this.dataPtr = dataPtr; - return this; - } - - public int getDataLen() { - return dataLen; - } - - public IndexBlock setDataLen(int dataLen) { - this.dataLen = dataLen; - return this; - } - - /** - * get the bytes for storage - * - * @return byte[] - */ - public byte[] getBytes() { - /* - * +------------+-----------+-----------+ - * | 4bytes | 4bytes | 4bytes | - * +------------+-----------+-----------+ - * start ip end ip data ptr + len - */ - byte[] b = new byte[12]; - - IpUtil.writeIntLong(b, 0, startIp); //start ip - IpUtil.writeIntLong(b, 4, endIp); //end ip - - //write the data ptr and the length - long mix = dataPtr | ((dataLen << 24) & 0xFF000000L); - IpUtil.writeIntLong(b, 8, mix); - - return b; - } -} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/Ip2Region.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/Ip2Region.java deleted file mode 100644 index 43b1eff..0000000 --- a/springboot-util/src/main/java/com/tiesheng/util/ip2region/Ip2Region.java +++ /dev/null @@ -1,416 +0,0 @@ -package com.tiesheng.util.ip2region; - -import cn.hutool.extra.spring.SpringUtil; -import com.tiesheng.util.config.Ip2regionConfig; - -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.charset.StandardCharsets; - -/** - * ip db searcher class (Not thread safe) - * - * @author chenxin - */ -public class Ip2Region { - - - private static Ip2Region ip2Region; - /** - * db config - */ - private final DbConfig dbConfig; - /** - * db file access handler - */ - private RandomAccessFile raf = null; - /** - * header blocks buffer - */ - private long[] headerSip = null; - private int[] headerPtr = null; - private int headerLength; - /** - * super blocks info - */ - private long firstIndexPtr = 0; - private long lastIndexPtr = 0; - private int totalIndexBlocks = 0; - /** - * for memory mode - * the original db binary string - */ - private byte[] dbBinStr = null; - - /** - * construct class - */ - public Ip2Region() { - this.dbConfig = new DbConfig(); - try { - Ip2regionConfig ip2regionConfig = SpringUtil.getBean(Ip2regionConfig.class); - raf = ip2regionConfig.getDbAccessFile(); - } catch (Exception ignored) { - } - } - - /** - * 获取单例 - * - * @return - */ - public static Ip2Region getInstance() { - if (ip2Region == null) { - synchronized (Ip2Region.class) { - if (ip2Region == null) { - ip2Region = new Ip2Region(); - } - } - } - return ip2Region; - } - - /** - * get the region with an int ip address with memory binary search algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock memorySearch(long ip) throws IOException { - int blen = IndexBlock.getIndexBlockLength(); - if (dbBinStr == null) { - dbBinStr = new byte[(int) raf.length()]; - raf.seek(0L); - raf.readFully(dbBinStr, 0, dbBinStr.length); - - //initialize the global vars - firstIndexPtr = IpUtil.getIntLong(dbBinStr, 0); - lastIndexPtr = IpUtil.getIntLong(dbBinStr, 4); - totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; - } - - //search the index blocks to define the data - int l = 0, h = totalIndexBlocks; - long sip, eip, dataptr = 0; - while (l <= h) { - int m = (l + h) >> 1; - int p = (int) (firstIndexPtr + m * blen); - - sip = IpUtil.getIntLong(dbBinStr, p); - if (ip < sip) { - h = m - 1; - } else { - eip = IpUtil.getIntLong(dbBinStr, p + 4); - if (ip > eip) { - l = m + 1; - } else { - dataptr = IpUtil.getIntLong(dbBinStr, p + 8); - break; - } - } - } - - //not matched - if (dataptr == 0) { - return null; - } - - //get the data - int dataLen = (int) ((dataptr >> 24) & 0xFF); - int dataPtr = (int) ((dataptr & 0x00FFFFFF)); - int cityId = (int) IpUtil.getIntLong(dbBinStr, dataPtr); - String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, StandardCharsets.UTF_8); - - return new DataBlock(cityId, region, dataPtr); - } - - /** - * get the region throught the ip address with memory binary search algorithm - * - * @return DataBlock - * @throws IOException - */ - public DataBlock memorySearch() throws IOException { - return memorySearch(null); - } - - /** - * get the region throught the ip address with memory binary search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock memorySearch(String ip) throws IOException { - return memorySearch(IpUtil.ip2long(ip)); - } - - - /** - * get by index ptr - * - * @param ptr - * @throws IOException - */ - public DataBlock getByIndexPtr(long ptr) throws IOException { - raf.seek(ptr); - byte[] buffer = new byte[12]; - raf.readFully(buffer, 0, buffer.length); - long extra = IpUtil.getIntLong(buffer, 8); - - int dataLen = (int) ((extra >> 24) & 0xFF); - int dataPtr = (int) ((extra & 0x00FFFFFF)); - - raf.seek(dataPtr); - byte[] data = new byte[dataLen]; - raf.readFully(data, 0, data.length); - - int cityId = (int) IpUtil.getIntLong(data, 0); - String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8); - - return new DataBlock(cityId, region, dataPtr); - } - - /** - * get the region with an int ip address with b-tree algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock btreeSearch(long ip) throws IOException { - //check and load the header - if (headerSip == null) { - raf.seek(8L); - byte[] b = new byte[dbConfig.getTotalHeaderSize()]; - raf.readFully(b, 0, b.length); - - //fill the header - int len = b.length >> 3, idx = 0; - headerSip = new long[len]; - headerPtr = new int[len]; - long startIp, dataPtr; - for (int i = 0; i < b.length; i += 8) { - startIp = IpUtil.getIntLong(b, i); - dataPtr = IpUtil.getIntLong(b, i + 4); - if (dataPtr == 0) { - break; - } - - headerSip[idx] = startIp; - headerPtr[idx] = (int) dataPtr; - idx++; - } - - headerLength = idx; - } - - //1. define the index block with the binary search - if (ip == headerSip[0]) { - return getByIndexPtr(headerPtr[0]); - } else if (ip == headerSip[headerLength - 1]) { - return getByIndexPtr(headerPtr[headerLength - 1]); - } - - int l = 0, h = headerLength, sptr = 0, eptr = 0; - while (l <= h) { - int m = (l + h) >> 1; - - //perfetc matched, just return it - if (ip == headerSip[m]) { - if (m > 0) { - sptr = headerPtr[m - 1]; - eptr = headerPtr[m]; - } else { - sptr = headerPtr[m]; - eptr = headerPtr[m + 1]; - } - - break; - } - - //less then the middle value - if (ip < headerSip[m]) { - if (m == 0) { - sptr = headerPtr[m]; - eptr = headerPtr[m + 1]; - break; - } else if (ip > headerSip[m - 1]) { - sptr = headerPtr[m - 1]; - eptr = headerPtr[m]; - break; - } - h = m - 1; - } else { - if (m == headerLength - 1) { - sptr = headerPtr[m - 1]; - eptr = headerPtr[m]; - break; - } else if (ip <= headerSip[m + 1]) { - sptr = headerPtr[m]; - eptr = headerPtr[m + 1]; - break; - } - l = m + 1; - } - } - - //match nothing just stop it - if (sptr == 0) { - return null; - } - - //2. search the index blocks to define the data - int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength(); - //include the right border block - byte[] iBuffer = new byte[blockLen + blen]; - raf.seek(sptr); - raf.readFully(iBuffer, 0, iBuffer.length); - - l = 0; - h = blockLen / blen; - long sip, eip, dataptr = 0; - while (l <= h) { - int m = (l + h) >> 1; - int p = m * blen; - sip = IpUtil.getIntLong(iBuffer, p); - if (ip < sip) { - h = m - 1; - } else { - eip = IpUtil.getIntLong(iBuffer, p + 4); - if (ip > eip) { - l = m + 1; - } else { - dataptr = IpUtil.getIntLong(iBuffer, p + 8); - break; - } - } - } - - //not matched - if (dataptr == 0) { - return null; - } - - //3. get the data - int dataLen = (int) ((dataptr >> 24) & 0xFF); - int dataPtr = (int) ((dataptr & 0x00FFFFFF)); - - raf.seek(dataPtr); - byte[] data = new byte[dataLen]; - raf.readFully(data, 0, data.length); - - int cityId = (int) IpUtil.getIntLong(data, 0); - String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8); - - return new DataBlock(cityId, region, dataPtr); - } - - /** - * get the region throught the ip address with b-tree search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock btreeSearch(String ip) { - try { - return btreeSearch(IpUtil.ip2long(ip)); - } catch (IOException ignored) { - } - return new DataBlock(0, "未知IP"); - } - - /** - * get the region with a int ip address with binary search algorithm - * - * @param ip - * @throws IOException - */ - public DataBlock binarySearch(long ip) throws IOException { - int blen = IndexBlock.getIndexBlockLength(); - if (totalIndexBlocks == 0) { - raf.seek(0L); - byte[] superBytes = new byte[8]; - raf.readFully(superBytes, 0, superBytes.length); - //initialize the global vars - firstIndexPtr = IpUtil.getIntLong(superBytes, 0); - lastIndexPtr = IpUtil.getIntLong(superBytes, 4); - totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1; - } - - //search the index blocks to define the data - int l = 0, h = totalIndexBlocks; - byte[] buffer = new byte[blen]; - long sip, eip, dataptr = 0; - while (l <= h) { - int m = (l + h) >> 1; - raf.seek(firstIndexPtr + m * blen); - raf.readFully(buffer, 0, buffer.length); - sip = IpUtil.getIntLong(buffer, 0); - if (ip < sip) { - h = m - 1; - } else { - eip = IpUtil.getIntLong(buffer, 4); - if (ip > eip) { - l = m + 1; - } else { - dataptr = IpUtil.getIntLong(buffer, 8); - break; - } - } - } - - //not matched - if (dataptr == 0) { - return null; - } - - //get the data - int dataLen = (int) ((dataptr >> 24) & 0xFF); - int dataPtr = (int) ((dataptr & 0x00FFFFFF)); - - raf.seek(dataPtr); - byte[] data = new byte[dataLen]; - raf.readFully(data, 0, data.length); - - int cityId = (int) IpUtil.getIntLong(data, 0); - String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8); - - return new DataBlock(cityId, region, dataPtr); - } - - /** - * get the region throught the ip address with binary search algorithm - * - * @param ip - * @return DataBlock - * @throws IOException - */ - public DataBlock binarySearch(String ip) throws IOException { - return binarySearch(IpUtil.ip2long(ip)); - } - - /** - * get the db config - * - * @return DbConfig - */ - public DbConfig getDbConfig() { - return dbConfig; - } - - /** - * close the db - * - * @throws IOException - */ - public void close() throws IOException { - headerSip = null; - headerPtr = null; - dbBinStr = null; - if (raf != null) { - raf.close(); - } - } - -} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/IpUtil.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/IpUtil.java deleted file mode 100644 index cab1e47..0000000 --- a/springboot-util/src/main/java/com/tiesheng/util/ip2region/IpUtil.java +++ /dev/null @@ -1,117 +0,0 @@ -package com.tiesheng.util.ip2region; - -/** - * util class - * - * @author chenxin - */ -public class IpUtil { - /** - * write specfield bytes to a byte array start from offset - * - * @param b - * @param offset - * @param v - * @param bytes - */ - public static void write(byte[] b, int offset, long v, int bytes) { - for (int i = 0; i < bytes; i++) { - b[offset++] = (byte) ((v >>> (8 * i)) & 0xFF); - } - } - - /** - * write a int to a byte array - * - * @param b - * @param offset - * @param v - */ - public static void writeIntLong(byte[] b, int offset, long v) { - b[offset++] = (byte) ((v >> 0) & 0xFF); - b[offset++] = (byte) ((v >> 8) & 0xFF); - b[offset++] = (byte) ((v >> 16) & 0xFF); - b[offset] = (byte) ((v >> 24) & 0xFF); - } - - /** - * get a int from a byte array start from the specifiled offset - * - * @param b - * @param offset - */ - public static long getIntLong(byte[] b, int offset) { - return ( - ((b[offset++] & 0x000000FFL)) | - ((b[offset++] << 8) & 0x0000FF00L) | - ((b[offset++] << 16) & 0x00FF0000L) | - ((b[offset] << 24) & 0xFF000000L) - ); - } - - /** - * get a int from a byte array start from the specifield offset - * - * @param b - * @param offset - */ - public static int getInt3(byte[] b, int offset) { - return ( - (b[offset++] & 0x000000FF) | - (b[offset++] & 0x0000FF00) | - (b[offset] & 0x00FF0000) - ); - } - - public static int getInt2(byte[] b, int offset) { - return ( - (b[offset++] & 0x000000FF) | - (b[offset] & 0x0000FF00) - ); - } - - public static int getInt1(byte[] b, int offset) { - return ( - (b[offset] & 0x000000FF) - ); - } - - /** - * string ip to long ip - * - * @param ip - * @return long - */ - public static long ip2long(String ip) { - String[] p = ip.split("\\."); - if (p.length != 4) { - return 0; - } - - int p1 = ((Integer.valueOf(p[0]) << 24) & 0xFF000000); - int p2 = ((Integer.valueOf(p[1]) << 16) & 0x00FF0000); - int p3 = ((Integer.valueOf(p[2]) << 8) & 0x0000FF00); - int p4 = ((Integer.valueOf(p[3]) << 0) & 0x000000FF); - - return ((p1 | p2 | p3 | p4) & 0xFFFFFFFFL); - } - - /** - * int to ip string - * - * @param ip - * @return string - */ - public static String long2ip(long ip) { - StringBuilder sb = new StringBuilder(); - - sb - .append((ip >> 24) & 0xFF).append('.') - .append((ip >> 16) & 0xFF).append('.') - .append((ip >> 8) & 0xFF).append('.') - .append((ip >> 0) & 0xFF); - - return sb.toString(); - } - -} diff --git a/springboot-util/src/main/java/com/tiesheng/util/ip2region/Searcher.java b/springboot-util/src/main/java/com/tiesheng/util/ip2region/Searcher.java new file mode 100644 index 0000000..4c43cc7 --- /dev/null +++ b/springboot-util/src/main/java/com/tiesheng/util/ip2region/Searcher.java @@ -0,0 +1,258 @@ +package com.tiesheng.util.ip2region; + +import java.io.IOException; +import java.io.RandomAccessFile; + +public class Searcher { + // constant defined copied from the xdb maker + public static final int HeaderInfoLength = 256; + public static final int VectorIndexRows = 256; + public static final int VectorIndexCols = 256; + public static final int VectorIndexSize = 8; + public static final int SegmentIndexSize = 14; + public static final byte[] shiftIndex = {24, 16, 8, 0}; + // random access file handle for file based search + private final RandomAccessFile handle; + // vector index. + // use the byte[] instead of VectorIndex entry array to keep + // the minimal memory allocation. + private final byte[] vectorIndex; + + // xdb content buffer, used for in-memory search + private final byte[] contentBuff; + + // --- static method to create searchers + private int ioCount = 0; + + public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException { + if (cBuff != null) { + this.handle = null; + this.vectorIndex = null; + this.contentBuff = cBuff; + } else { + this.handle = new RandomAccessFile(dbFile, "r"); + this.vectorIndex = vectorIndex; + this.contentBuff = null; + } + } + + public static Searcher newWithFileOnly(String dbPath) throws IOException { + return new Searcher(dbPath, null, null); + } + + // --- End of creator + + public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException { + return new Searcher(dbPath, vectorIndex, null); + } + + public static Searcher newWithBuffer(byte[] cBuff) throws IOException { + return new Searcher(null, null, cBuff); + } + + public static Header loadHeader(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[HeaderInfoLength]; + handle.read(buff); + return new Header(buff); + } + + public static Header loadHeaderFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final Header header = loadHeader(handle); + handle.close(); + return header; + } + + public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException { + handle.seek(HeaderInfoLength); + int len = VectorIndexRows * VectorIndexCols * VectorIndexSize; + final byte[] buff = new byte[len]; + int rLen = handle.read(buff); + if (rLen != len) { + throw new IOException("incomplete read: read bytes should be " + len); + } + + return buff; + } + + public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final byte[] vIndex = loadVectorIndex(handle); + handle.close(); + return vIndex; + } + + // --- static cache util function + + public static byte[] loadContent(RandomAccessFile handle) throws IOException { + handle.seek(0); + final byte[] buff = new byte[(int) handle.length()]; + int rLen = handle.read(buff); + if (rLen != buff.length) { + throw new IOException("incomplete read: read bytes should be " + buff.length); + } + + return buff; + } + + public static byte[] loadContentFromFile(String dbPath) throws IOException { + final RandomAccessFile handle = new RandomAccessFile(dbPath, "r"); + final byte[] content = loadContent(handle); + handle.close(); + return content; + } + + /* get an int from a byte array start from the specified offset */ + public static long getIntLong(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FFL)) | + ((b[offset++] << 8) & 0x0000FF00L) | + ((b[offset++] << 16) & 0x00FF0000L) | + ((b[offset] << 24) & 0xFF000000L) + ); + } + + public static int getInt(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset++] << 8) & 0x0000FF00) | + ((b[offset++] << 16) & 0x00FF0000) | + ((b[offset] << 24) & 0xFF000000) + ); + } + + public static int getInt2(byte[] b, int offset) { + return ( + ((b[offset++] & 0x000000FF)) | + ((b[offset] << 8) & 0x0000FF00) + ); + } + + /* long int to ip string */ + public static String long2ip(long ip) { + return String.valueOf((ip >> 24) & 0xFF) + '.' + + ((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF); + } + + // --- End cache load util function + + // --- static util method + + /* check the specified ip address */ + public static long checkIP(String ip) throws Exception { + String[] ps = ip.split("\\."); + if (ps.length != 4) { + throw new Exception("invalid ip address `" + ip + "`"); + } + + long ipDst = 0; + for (int i = 0; i < ps.length; i++) { + int val = Integer.parseInt(ps[i]); + if (val > 255) { + throw new Exception("ip part `" + ps[i] + "` should be less then 256"); + } + + ipDst |= ((long) val << shiftIndex[i]); + } + + return ipDst & 0xFFFFFFFFL; + } + + public void close() throws IOException { + if (this.handle != null) { + this.handle.close(); + } + } + + public int getIOCount() { + return ioCount; + } + + public String search(String ipStr) throws Exception { + long ip = checkIP(ipStr); + return search(ip); + } + + public String search(long ip) throws IOException { + // reset the global counter + this.ioCount = 0; + + // locate the segment index block based on the vector index + int sPtr = 0, ePtr = 0; + int il0 = (int) ((ip >> 24) & 0xFF); + int il1 = (int) ((ip >> 16) & 0xFF); + int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize; + // System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx); + if (vectorIndex != null) { + sPtr = getInt(vectorIndex, idx); + ePtr = getInt(vectorIndex, idx + 4); + } else if (contentBuff != null) { + sPtr = getInt(contentBuff, HeaderInfoLength + idx); + ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4); + } else { + final byte[] buff = new byte[VectorIndexSize]; + read(HeaderInfoLength + idx, buff); + sPtr = getInt(buff, 0); + ePtr = getInt(buff, 4); + } + + // System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr); + + // binary search the segment index block to get the region info + final byte[] buff = new byte[SegmentIndexSize]; + int dataLen = -1, dataPtr = -1; + int l = 0, h = (ePtr - sPtr) / SegmentIndexSize; + while (l <= h) { + int m = (l + h) >> 1; + int p = sPtr + m * SegmentIndexSize; + + // read the segment index + read(p, buff); + long sip = getIntLong(buff, 0); + if (ip < sip) { + h = m - 1; + } else { + long eip = getIntLong(buff, 4); + if (ip > eip) { + l = m + 1; + } else { + dataLen = getInt2(buff, 8); + dataPtr = getInt(buff, 10); + break; + } + } + } + + // empty match interception + // System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr); + if (dataPtr < 0) { + return null; + } + + // load and return the region data + final byte[] regionBuff = new byte[dataLen]; + read(dataPtr, regionBuff); + return new String(regionBuff, "utf-8"); + } + + protected void read(int offset, byte[] buffer) throws IOException { + // check the in-memory buffer first + if (contentBuff != null) { + // @TODO: reduce data copying, directly decode the data ? + System.arraycopy(contentBuff, offset, buffer, 0, buffer.length); + return; + } + + // read from the file handle + assert handle != null; + handle.seek(offset); + + this.ioCount++; + int rLen = handle.read(buffer); + if (rLen != buffer.length) { + throw new IOException("incomplete read: read bytes should be " + buffer.length); + } + } + +} diff --git a/springboot-web/src/main/java/com/tiesheng/core/service/CoreLogService.java b/springboot-web/src/main/java/com/tiesheng/core/service/CoreLogService.java index 32b0212..b4a9070 100644 --- a/springboot-web/src/main/java/com/tiesheng/core/service/CoreLogService.java +++ b/springboot-web/src/main/java/com/tiesheng/core/service/CoreLogService.java @@ -16,8 +16,7 @@ import com.tiesheng.login.config.token.bean.TokenBean; import com.tiesheng.message.pojos.MessageReqResp; import com.tiesheng.message.service.TieshengMessageConfigurer; import com.tiesheng.util.ServletKit; -import com.tiesheng.util.ip2region.DataBlock; -import com.tiesheng.util.ip2region.Ip2Region; +import com.tiesheng.util.config.Ip2regionConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -35,6 +34,8 @@ public class CoreLogService extends TsServiceBase