tieshengkeji/tiesheng-springboot
1
1
Fork 0
Code Pull Requests Actions Packages Releases Activity

perf：升级Ip2Region

Browse Source
This commit is contained in:
曾文豪
2023-02-20 14:28:13 +08:00
parent b79895aa56
commit 7fcc407514
10 changed files with 321 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
springboot-ademo/src/main/java/com/tiesheng/demo/controller/TestController.java
View File

@@ -8,6 +8,7 @@ import com.tiesheng.login.config.token.bean.TokenBean;
import com.tiesheng.message.config.aliyun.AliyunSmsConfig;
import com.tiesheng.message.pojos.MessageReqResp;
import com.tiesheng.util.config.GlobalConfig;
import com.tiesheng.util.config.Ip2regionConfig;
import com.tiesheng.util.pojos.ApiResp;
import com.tiesheng.util.pojos.FileUploadPath;
import org.springframework.beans.factory.annotation.Autowired;
@@ -29,6 +30,9 @@ public class TestController {
AliyunSmsConfig aliyunSmsConfig;
@Autowired
TsTokenConfig tsTokenConfig;
@Autowired
Ip2regionConfig ip2regionConfig;
@RequestMapping("/index")
@TokenIgnore
@@ -80,4 +84,12 @@ public class TestController {
return ApiResp.respOK(uploadPath.getAbsolutePath());
}
@RequestMapping("searchIP")
@TokenIgnore
public ApiResp<String> searchIp() {
String search = ip2regionConfig.search("127.0.0.1");
return ApiResp.respOK(search);
}
}

22
springboot-util/src/main/java/com/tiesheng/util/config/Ip2regionConfig.java
View File

@@ -3,12 +3,11 @@ package com.tiesheng.util.config;
import cn.hutool.core.io.FileUtil;
import cn.hutool.http.HttpUtil;
import cn.hutool.log.LogFactory;
import com.tiesheng.util.ip2region.Searcher;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import javax.annotation.PostConstruct;
import java.io.FileNotFoundException;
import java.io.RandomAccessFile;
/**
* @author hao
@@ -17,8 +16,8 @@ import java.io.RandomAccessFile;
@ConfigurationProperties(prefix = "tiesheng.ip2region")
public class Ip2regionConfig {
private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.db";
private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.db";
private String dbUrl = "http://git.kepai365.com/zeng_wenhao/kepai-repo/raw/master/ipdb/ip2region.xdb";
private String dbPath = System.getProperty("user.dir") + "/runtime/ip2region/ip2region.xdb";
///////////////////////////////////////////////////////////////////////////
// 逻辑方法
@@ -35,15 +34,20 @@ public class Ip2regionConfig {
/**
* 获取db文件
* 搜索ip
*
* @param ip
* @return
*/
public RandomAccessFile getDbAccessFile() throws FileNotFoundException {
if (!FileUtil.exist(dbPath)) {
downloadDbFile();
public String search(String ip) {
String region = "";
try {
Searcher searcher = Searcher.newWithFileOnly(dbPath);
region = searcher.search(ip);
searcher.close();
} catch (Exception ignored) {
}
return new RandomAccessFile(dbPath, "r");
return region;
}
///////////////////////////////////////////////////////////////////////////

76
springboot-util/src/main/java/com/tiesheng/util/ip2region/DataBlock.java
View File

@@ -1,76 +0,0 @@
package com.tiesheng.util.ip2region;
/**
* data block class
*
* @author chenxin<chenxin619315 @ gmail.com>
*/
public class DataBlock {
/**
* city id
*/
private int cityId;
/**
* region address
*/
private String region;
/**
* region ptr in the db file
*/
private int dataPtr;
/**
* construct method
*
* @param cityId
* @param region region string
* @param dataPtr data ptr
*/
public DataBlock(int cityId, String region, int dataPtr) {
this.cityId = cityId;
this.region = region;
this.dataPtr = dataPtr;
}
public DataBlock(int cityId, String region) {
this(cityId, region, 0);
}
public int getCityId() {
return cityId;
}
public DataBlock setCityId(int cityId) {
this.cityId = cityId;
return this;
}
public String getRegion() {
return region;
}
public DataBlock setRegion(String region) {
this.region = region;
return this;
}
public int getDataPtr() {
return dataPtr;
}
public DataBlock setDataPtr(int dataPtr) {
this.dataPtr = dataPtr;
return this;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(cityId).append('|').append(region).append('|').append(dataPtr);
return sb.toString();
}
}

51
springboot-util/src/main/java/com/tiesheng/util/ip2region/DbConfig.java
View File

@@ -1,51 +0,0 @@
package com.tiesheng.util.ip2region;
/**
* database configuration class
*
* @author chenxin<chenxin619315 @ gmail.com>
*/
public class DbConfig {
/**
* total header data block size
*/
private int totalHeaderSize;
/**
* max index data block size
* u should always choice the fastest read block size
*/
private int indexBlockSize;
/**
* construct method
*
* @param totalHeaderSize
*/
public DbConfig(int totalHeaderSize) {
this.totalHeaderSize = totalHeaderSize;
this.indexBlockSize = 8192;
}
public DbConfig() {
this(8 * 2048);
}
public int getTotalHeaderSize() {
return totalHeaderSize;
}
public DbConfig setTotalHeaderSize(int totalHeaderSize) {
this.totalHeaderSize = totalHeaderSize;
return this;
}
public int getIndexBlockSize() {
return indexBlockSize;
}
public DbConfig setIndexBlockSize(int dataBlockSize) {
this.indexBlockSize = dataBlockSize;
return this;
}
}

34
springboot-util/src/main/java/com/tiesheng/util/ip2region/Header.java Normal file
View File

@@ -0,0 +1,34 @@
package com.tiesheng.util.ip2region;
/**
* @author hao
*/
public class Header {
public final int version;
public final int indexPolicy;
public final int createdAt;
public final int startIndexPtr;
public final int endIndexPtr;
public final byte[] buffer;
public Header(byte[] buff) {
assert buff.length >= 16;
version = Searcher.getInt2(buff, 0);
indexPolicy = Searcher.getInt2(buff, 2);
createdAt = Searcher.getInt(buff, 4);
startIndexPtr = Searcher.getInt(buff, 8);
endIndexPtr = Searcher.getInt(buff, 12);
buffer = buff;
}
@Override
public String toString() {
return "{" +
"Version: " + version + ',' +
"IndexPolicy: " + indexPolicy + ',' +
"CreatedAt: " + createdAt + ',' +
"StartIndexPtr: " + startIndexPtr + ',' +
"EndIndexPtr: " + endIndexPtr +
'}';
}
}

102
springboot-util/src/main/java/com/tiesheng/util/ip2region/IndexBlock.java
View File

@@ -1,102 +0,0 @@
package com.tiesheng.util.ip2region;
/**
* item index class
*
* @author chenxin<chenxin619315 @ gmail.com>
*/
public class IndexBlock {
private static int LENGTH = 12;
/**
* start ip address
*/
private long startIp;
/**
* end ip address
*/
private long endIp;
/**
* data ptr and data length
*/
private int dataPtr;
/**
* data length
*/
private int dataLen;
public IndexBlock(long startIp, long endIp, int dataPtr, int dataLen) {
this.startIp = startIp;
this.endIp = endIp;
this.dataPtr = dataPtr;
this.dataLen = dataLen;
}
public static int getIndexBlockLength() {
return LENGTH;
}
public long getStartIp() {
return startIp;
}
public IndexBlock setStartIp(long startIp) {
this.startIp = startIp;
return this;
}
public long getEndIp() {
return endIp;
}
public IndexBlock setEndIp(long endIp) {
this.endIp = endIp;
return this;
}
public int getDataPtr() {
return dataPtr;
}
public IndexBlock setDataPtr(int dataPtr) {
this.dataPtr = dataPtr;
return this;
}
public int getDataLen() {
return dataLen;
}
public IndexBlock setDataLen(int dataLen) {
this.dataLen = dataLen;
return this;
}
/**
* get the bytes for storage
*
* @return byte[]
*/
public byte[] getBytes() {
/*
* +------------+-----------+-----------+
* | 4bytes | 4bytes | 4bytes |
* +------------+-----------+-----------+
* start ip end ip data ptr + len
*/
byte[] b = new byte[12];
IpUtil.writeIntLong(b, 0, startIp); //start ip
IpUtil.writeIntLong(b, 4, endIp); //end ip
//write the data ptr and the length
long mix = dataPtr | ((dataLen << 24) & 0xFF000000L);
IpUtil.writeIntLong(b, 8, mix);
return b;
}
}

416
springboot-util/src/main/java/com/tiesheng/util/ip2region/Ip2Region.java
View File

@@ -1,416 +0,0 @@
package com.tiesheng.util.ip2region;
import cn.hutool.extra.spring.SpringUtil;
import com.tiesheng.util.config.Ip2regionConfig;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.charset.StandardCharsets;
/**
* ip db searcher class (Not thread safe)
*
* @author chenxin<chenxin619315 @ gmail.com>
*/
public class Ip2Region {
private static Ip2Region ip2Region;
/**
* db config
*/
private final DbConfig dbConfig;
/**
* db file access handler
*/
private RandomAccessFile raf = null;
/**
* header blocks buffer
*/
private long[] headerSip = null;
private int[] headerPtr = null;
private int headerLength;
/**
* super blocks info
*/
private long firstIndexPtr = 0;
private long lastIndexPtr = 0;
private int totalIndexBlocks = 0;
/**
* for memory mode
* the original db binary string
*/
private byte[] dbBinStr = null;
/**
* construct class
*/
public Ip2Region() {
this.dbConfig = new DbConfig();
try {
Ip2regionConfig ip2regionConfig = SpringUtil.getBean(Ip2regionConfig.class);
raf = ip2regionConfig.getDbAccessFile();
} catch (Exception ignored) {
}
}
/**
* 获取单例
*
* @return
*/
public static Ip2Region getInstance() {
if (ip2Region == null) {
synchronized (Ip2Region.class) {
if (ip2Region == null) {
ip2Region = new Ip2Region();
}
}
}
return ip2Region;
}
/**
* get the region with an int ip address with memory binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock memorySearch(long ip) throws IOException {
int blen = IndexBlock.getIndexBlockLength();
if (dbBinStr == null) {
dbBinStr = new byte[(int) raf.length()];
raf.seek(0L);
raf.readFully(dbBinStr, 0, dbBinStr.length);
//initialize the global vars
firstIndexPtr = IpUtil.getIntLong(dbBinStr, 0);
lastIndexPtr = IpUtil.getIntLong(dbBinStr, 4);
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
long sip, eip, dataptr = 0;
while (l <= h) {
int m = (l + h) >> 1;
int p = (int) (firstIndexPtr + m * blen);
sip = IpUtil.getIntLong(dbBinStr, p);
if (ip < sip) {
h = m - 1;
} else {
eip = IpUtil.getIntLong(dbBinStr, p + 4);
if (ip > eip) {
l = m + 1;
} else {
dataptr = IpUtil.getIntLong(dbBinStr, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0) {
return null;
}
//get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
int cityId = (int) IpUtil.getIntLong(dbBinStr, dataPtr);
String region = new String(dbBinStr, dataPtr + 4, dataLen - 4, StandardCharsets.UTF_8);
return new DataBlock(cityId, region, dataPtr);
}
/**
* get the region throught the ip address with memory binary search algorithm
*
* @return DataBlock
* @throws IOException
*/
public DataBlock memorySearch() throws IOException {
return memorySearch(null);
}
/**
* get the region throught the ip address with memory binary search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock memorySearch(String ip) throws IOException {
return memorySearch(IpUtil.ip2long(ip));
}
/**
* get by index ptr
*
* @param ptr
* @throws IOException
*/
public DataBlock getByIndexPtr(long ptr) throws IOException {
raf.seek(ptr);
byte[] buffer = new byte[12];
raf.readFully(buffer, 0, buffer.length);
long extra = IpUtil.getIntLong(buffer, 8);
int dataLen = (int) ((extra >> 24) & 0xFF);
int dataPtr = (int) ((extra & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.length);
int cityId = (int) IpUtil.getIntLong(data, 0);
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
return new DataBlock(cityId, region, dataPtr);
}
/**
* get the region with an int ip address with b-tree algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock btreeSearch(long ip) throws IOException {
//check and load the header
if (headerSip == null) {
raf.seek(8L);
byte[] b = new byte[dbConfig.getTotalHeaderSize()];
raf.readFully(b, 0, b.length);
//fill the header
int len = b.length >> 3, idx = 0;
headerSip = new long[len];
headerPtr = new int[len];
long startIp, dataPtr;
for (int i = 0; i < b.length; i += 8) {
startIp = IpUtil.getIntLong(b, i);
dataPtr = IpUtil.getIntLong(b, i + 4);
if (dataPtr == 0) {
break;
}
headerSip[idx] = startIp;
headerPtr[idx] = (int) dataPtr;
idx++;
}
headerLength = idx;
}
//1. define the index block with the binary search
if (ip == headerSip[0]) {
return getByIndexPtr(headerPtr[0]);
} else if (ip == headerSip[headerLength - 1]) {
return getByIndexPtr(headerPtr[headerLength - 1]);
}
int l = 0, h = headerLength, sptr = 0, eptr = 0;
while (l <= h) {
int m = (l + h) >> 1;
//perfetc matched, just return it
if (ip == headerSip[m]) {
if (m > 0) {
sptr = headerPtr[m - 1];
eptr = headerPtr[m];
} else {
sptr = headerPtr[m];
eptr = headerPtr[m + 1];
}
break;
}
//less then the middle value
if (ip < headerSip[m]) {
if (m == 0) {
sptr = headerPtr[m];
eptr = headerPtr[m + 1];
break;
} else if (ip > headerSip[m - 1]) {
sptr = headerPtr[m - 1];
eptr = headerPtr[m];
break;
}
h = m - 1;
} else {
if (m == headerLength - 1) {
sptr = headerPtr[m - 1];
eptr = headerPtr[m];
break;
} else if (ip <= headerSip[m + 1]) {
sptr = headerPtr[m];
eptr = headerPtr[m + 1];
break;
}
l = m + 1;
}
}
//match nothing just stop it
if (sptr == 0) {
return null;
}
//2. search the index blocks to define the data
int blockLen = eptr - sptr, blen = IndexBlock.getIndexBlockLength();
//include the right border block
byte[] iBuffer = new byte[blockLen + blen];
raf.seek(sptr);
raf.readFully(iBuffer, 0, iBuffer.length);
l = 0;
h = blockLen / blen;
long sip, eip, dataptr = 0;
while (l <= h) {
int m = (l + h) >> 1;
int p = m * blen;
sip = IpUtil.getIntLong(iBuffer, p);
if (ip < sip) {
h = m - 1;
} else {
eip = IpUtil.getIntLong(iBuffer, p + 4);
if (ip > eip) {
l = m + 1;
} else {
dataptr = IpUtil.getIntLong(iBuffer, p + 8);
break;
}
}
}
//not matched
if (dataptr == 0) {
return null;
}
//3. get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.length);
int cityId = (int) IpUtil.getIntLong(data, 0);
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
return new DataBlock(cityId, region, dataPtr);
}
/**
* get the region throught the ip address with b-tree search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock btreeSearch(String ip) {
try {
return btreeSearch(IpUtil.ip2long(ip));
} catch (IOException ignored) {
}
return new DataBlock(0, "未知IP");
}
/**
* get the region with a int ip address with binary search algorithm
*
* @param ip
* @throws IOException
*/
public DataBlock binarySearch(long ip) throws IOException {
int blen = IndexBlock.getIndexBlockLength();
if (totalIndexBlocks == 0) {
raf.seek(0L);
byte[] superBytes = new byte[8];
raf.readFully(superBytes, 0, superBytes.length);
//initialize the global vars
firstIndexPtr = IpUtil.getIntLong(superBytes, 0);
lastIndexPtr = IpUtil.getIntLong(superBytes, 4);
totalIndexBlocks = (int) ((lastIndexPtr - firstIndexPtr) / blen) + 1;
}
//search the index blocks to define the data
int l = 0, h = totalIndexBlocks;
byte[] buffer = new byte[blen];
long sip, eip, dataptr = 0;
while (l <= h) {
int m = (l + h) >> 1;
raf.seek(firstIndexPtr + m * blen);
raf.readFully(buffer, 0, buffer.length);
sip = IpUtil.getIntLong(buffer, 0);
if (ip < sip) {
h = m - 1;
} else {
eip = IpUtil.getIntLong(buffer, 4);
if (ip > eip) {
l = m + 1;
} else {
dataptr = IpUtil.getIntLong(buffer, 8);
break;
}
}
}
//not matched
if (dataptr == 0) {
return null;
}
//get the data
int dataLen = (int) ((dataptr >> 24) & 0xFF);
int dataPtr = (int) ((dataptr & 0x00FFFFFF));
raf.seek(dataPtr);
byte[] data = new byte[dataLen];
raf.readFully(data, 0, data.length);
int cityId = (int) IpUtil.getIntLong(data, 0);
String region = new String(data, 4, data.length - 4, StandardCharsets.UTF_8);
return new DataBlock(cityId, region, dataPtr);
}
/**
* get the region throught the ip address with binary search algorithm
*
* @param ip
* @return DataBlock
* @throws IOException
*/
public DataBlock binarySearch(String ip) throws IOException {
return binarySearch(IpUtil.ip2long(ip));
}
/**
* get the db config
*
* @return DbConfig
*/
public DbConfig getDbConfig() {
return dbConfig;
}
/**
* close the db
*
* @throws IOException
*/
public void close() throws IOException {
headerSip = null;
headerPtr = null;
dbBinStr = null;
if (raf != null) {
raf.close();
}
}
}

117
springboot-util/src/main/java/com/tiesheng/util/ip2region/IpUtil.java
View File

@@ -1,117 +0,0 @@
package com.tiesheng.util.ip2region;
/**
* util class
*
* @author chenxin<chenxin619315 @ gmail.com>
*/
public class IpUtil {
/**
* write specfield bytes to a byte array start from offset
*
* @param b
* @param offset
* @param v
* @param bytes
*/
public static void write(byte[] b, int offset, long v, int bytes) {
for (int i = 0; i < bytes; i++) {
b[offset++] = (byte) ((v >>> (8 * i)) & 0xFF);
}
}
/**
* write a int to a byte array
*
* @param b
* @param offset
* @param v
*/
public static void writeIntLong(byte[] b, int offset, long v) {
b[offset++] = (byte) ((v >> 0) & 0xFF);
b[offset++] = (byte) ((v >> 8) & 0xFF);
b[offset++] = (byte) ((v >> 16) & 0xFF);
b[offset] = (byte) ((v >> 24) & 0xFF);
}
/**
* get a int from a byte array start from the specifiled offset
*
* @param b
* @param offset
*/
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset] << 24) & 0xFF000000L)
);
}
/**
* get a int from a byte array start from the specifield offset
*
* @param b
* @param offset
*/
public static int getInt3(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset++] & 0x0000FF00) |
(b[offset] & 0x00FF0000)
);
}
public static int getInt2(byte[] b, int offset) {
return (
(b[offset++] & 0x000000FF) |
(b[offset] & 0x0000FF00)
);
}
public static int getInt1(byte[] b, int offset) {
return (
(b[offset] & 0x000000FF)
);
}
/**
* string ip to long ip
*
* @param ip
* @return long
*/
public static long ip2long(String ip) {
String[] p = ip.split("\\.");
if (p.length != 4) {
return 0;
}
int p1 = ((Integer.valueOf(p[0]) << 24) & 0xFF000000);
int p2 = ((Integer.valueOf(p[1]) << 16) & 0x00FF0000);
int p3 = ((Integer.valueOf(p[2]) << 8) & 0x0000FF00);
int p4 = ((Integer.valueOf(p[3]) << 0) & 0x000000FF);
return ((p1 | p2 | p3 | p4) & 0xFFFFFFFFL);
}
/**
* int to ip string
*
* @param ip
* @return string
*/
public static String long2ip(long ip) {
StringBuilder sb = new StringBuilder();
sb
.append((ip >> 24) & 0xFF).append('.')
.append((ip >> 16) & 0xFF).append('.')
.append((ip >> 8) & 0xFF).append('.')
.append((ip >> 0) & 0xFF);
return sb.toString();
}
}

258
springboot-util/src/main/java/com/tiesheng/util/ip2region/Searcher.java Normal file
View File

@@ -0,0 +1,258 @@
package com.tiesheng.util.ip2region;
import java.io.IOException;
import java.io.RandomAccessFile;
public class Searcher {
// constant defined copied from the xdb maker
public static final int HeaderInfoLength = 256;
public static final int VectorIndexRows = 256;
public static final int VectorIndexCols = 256;
public static final int VectorIndexSize = 8;
public static final int SegmentIndexSize = 14;
public static final byte[] shiftIndex = {24, 16, 8, 0};
// random access file handle for file based search
private final RandomAccessFile handle;
// vector index.
// use the byte[] instead of VectorIndex entry array to keep
// the minimal memory allocation.
private final byte[] vectorIndex;
// xdb content buffer, used for in-memory search
private final byte[] contentBuff;
// --- static method to create searchers
private int ioCount = 0;
public Searcher(String dbFile, byte[] vectorIndex, byte[] cBuff) throws IOException {
if (cBuff != null) {
this.handle = null;
this.vectorIndex = null;
this.contentBuff = cBuff;
} else {
this.handle = new RandomAccessFile(dbFile, "r");
this.vectorIndex = vectorIndex;
this.contentBuff = null;
}
}
public static Searcher newWithFileOnly(String dbPath) throws IOException {
return new Searcher(dbPath, null, null);
}
// --- End of creator
public static Searcher newWithVectorIndex(String dbPath, byte[] vectorIndex) throws IOException {
return new Searcher(dbPath, vectorIndex, null);
}
public static Searcher newWithBuffer(byte[] cBuff) throws IOException {
return new Searcher(null, null, cBuff);
}
public static Header loadHeader(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[HeaderInfoLength];
handle.read(buff);
return new Header(buff);
}
public static Header loadHeaderFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final Header header = loadHeader(handle);
handle.close();
return header;
}
public static byte[] loadVectorIndex(RandomAccessFile handle) throws IOException {
handle.seek(HeaderInfoLength);
int len = VectorIndexRows * VectorIndexCols * VectorIndexSize;
final byte[] buff = new byte[len];
int rLen = handle.read(buff);
if (rLen != len) {
throw new IOException("incomplete read: read bytes should be " + len);
}
return buff;
}
public static byte[] loadVectorIndexFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] vIndex = loadVectorIndex(handle);
handle.close();
return vIndex;
}
// --- static cache util function
public static byte[] loadContent(RandomAccessFile handle) throws IOException {
handle.seek(0);
final byte[] buff = new byte[(int) handle.length()];
int rLen = handle.read(buff);
if (rLen != buff.length) {
throw new IOException("incomplete read: read bytes should be " + buff.length);
}
return buff;
}
public static byte[] loadContentFromFile(String dbPath) throws IOException {
final RandomAccessFile handle = new RandomAccessFile(dbPath, "r");
final byte[] content = loadContent(handle);
handle.close();
return content;
}
/* get an int from a byte array start from the specified offset */
public static long getIntLong(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FFL)) |
((b[offset++] << 8) & 0x0000FF00L) |
((b[offset++] << 16) & 0x00FF0000L) |
((b[offset] << 24) & 0xFF000000L)
);
}
public static int getInt(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FF)) |
((b[offset++] << 8) & 0x0000FF00) |
((b[offset++] << 16) & 0x00FF0000) |
((b[offset] << 24) & 0xFF000000)
);
}
public static int getInt2(byte[] b, int offset) {
return (
((b[offset++] & 0x000000FF)) |
((b[offset] << 8) & 0x0000FF00)
);
}
/* long int to ip string */
public static String long2ip(long ip) {
return String.valueOf((ip >> 24) & 0xFF) + '.' +
((ip >> 16) & 0xFF) + '.' + ((ip >> 8) & 0xFF) + '.' + ((ip) & 0xFF);
}
// --- End cache load util function
// --- static util method
/* check the specified ip address */
public static long checkIP(String ip) throws Exception {
String[] ps = ip.split("\\.");
if (ps.length != 4) {
throw new Exception("invalid ip address `" + ip + "`");
}
long ipDst = 0;
for (int i = 0; i < ps.length; i++) {
int val = Integer.parseInt(ps[i]);
if (val > 255) {
throw new Exception("ip part `" + ps[i] + "` should be less then 256");
}
ipDst |= ((long) val << shiftIndex[i]);
}
return ipDst & 0xFFFFFFFFL;
}
public void close() throws IOException {
if (this.handle != null) {
this.handle.close();
}
}
public int getIOCount() {
return ioCount;
}
public String search(String ipStr) throws Exception {
long ip = checkIP(ipStr);
return search(ip);
}
public String search(long ip) throws IOException {
// reset the global counter
this.ioCount = 0;
// locate the segment index block based on the vector index
int sPtr = 0, ePtr = 0;
int il0 = (int) ((ip >> 24) & 0xFF);
int il1 = (int) ((ip >> 16) & 0xFF);
int idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize;
// System.out.printf("il0: %d, il1: %d, idx: %d\n", il0, il1, idx);
if (vectorIndex != null) {
sPtr = getInt(vectorIndex, idx);
ePtr = getInt(vectorIndex, idx + 4);
} else if (contentBuff != null) {
sPtr = getInt(contentBuff, HeaderInfoLength + idx);
ePtr = getInt(contentBuff, HeaderInfoLength + idx + 4);
} else {
final byte[] buff = new byte[VectorIndexSize];
read(HeaderInfoLength + idx, buff);
sPtr = getInt(buff, 0);
ePtr = getInt(buff, 4);
}
// System.out.printf("sPtr: %d, ePtr: %d\n", sPtr, ePtr);
// binary search the segment index block to get the region info
final byte[] buff = new byte[SegmentIndexSize];
int dataLen = -1, dataPtr = -1;
int l = 0, h = (ePtr - sPtr) / SegmentIndexSize;
while (l <= h) {
int m = (l + h) >> 1;
int p = sPtr + m * SegmentIndexSize;
// read the segment index
read(p, buff);
long sip = getIntLong(buff, 0);
if (ip < sip) {
h = m - 1;
} else {
long eip = getIntLong(buff, 4);
if (ip > eip) {
l = m + 1;
} else {
dataLen = getInt2(buff, 8);
dataPtr = getInt(buff, 10);
break;
}
}
}
// empty match interception
// System.out.printf("dataLen: %d, dataPtr: %d\n", dataLen, dataPtr);
if (dataPtr < 0) {
return null;
}
// load and return the region data
final byte[] regionBuff = new byte[dataLen];
read(dataPtr, regionBuff);
return new String(regionBuff, "utf-8");
}
protected void read(int offset, byte[] buffer) throws IOException {
// check the in-memory buffer first
if (contentBuff != null) {
// @TODO: reduce data copying, directly decode the data ?
System.arraycopy(contentBuff, offset, buffer, 0, buffer.length);
return;
}
// read from the file handle
assert handle != null;
handle.seek(offset);
this.ioCount++;
int rLen = handle.read(buffer);
if (rLen != buffer.length) {
throw new IOException("incomplete read: read bytes should be " + buffer.length);
}
}
}

8
springboot-web/src/main/java/com/tiesheng/core/service/CoreLogService.java
View File

@@ -16,8 +16,7 @@ import com.tiesheng.login.config.token.bean.TokenBean;
import com.tiesheng.message.pojos.MessageReqResp;
import com.tiesheng.message.service.TieshengMessageConfigurer;
import com.tiesheng.util.ServletKit;
import com.tiesheng.util.ip2region.DataBlock;
import com.tiesheng.util.ip2region.Ip2Region;
import com.tiesheng.util.config.Ip2regionConfig;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@@ -35,6 +34,8 @@ public class CoreLogService extends TsServiceBase<CoreLogOperationMapper, CoreLo
CoreLogLoginMapper coreLogLoginMapper;
@Autowired
CoreLogMessageMapper coreLogMessageMapper;
@Autowired
Ip2regionConfig ip2regionConfig;
public CoreLogLoginMapper getLogLoginMapper() {
return coreLogLoginMapper;
@@ -88,8 +89,7 @@ public class CoreLogService extends TsServiceBase<CoreLogOperationMapper, CoreLo
login.setUserName(currentWebUser.getName());
login.setIp(ip);
DataBlock dataBlock = Ip2Region.getInstance().btreeSearch(ip);
login.setAddress(dataBlock.getRegion());
login.setAddress(ip2regionConfig.search(login.getIp()));
coreLogLoginMapper.insert(login);
}