rebar3 服务器
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
.rebar3
|
||||
_*
|
||||
.eunit
|
||||
*.o
|
||||
*.beam
|
||||
*.plt
|
||||
*.swp
|
||||
*.swo
|
||||
.erlang.cookie
|
||||
ebin
|
||||
log
|
||||
erl_crash.dump
|
||||
.rebar
|
||||
logs
|
||||
_build
|
||||
.idea
|
||||
*.iml
|
||||
rebar3.crashdump
|
||||
*~
|
||||
@@ -0,0 +1,191 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
Copyright 2021, Anonymous <anonymous@example.org>.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
**1、ms_cache服务**
|
||||
ms_cache是居于第三方mysql驱动开发的缓存功能；支持游戏节点内部署，做为游戏服的一部分；也支持独立部署，作为数据中心使用。
|
||||
ms_cache使用会涉及到三个比较重要的东西
|
||||
1、ms_cache的application的配置，指定ms_cache使用的数据库地址、类型、运作方式等
|
||||
2、rebar3_cache_plugin插件工具，通过配置文件生成ms_cache缓存服务需要的适配文件
|
||||
3、rebar3_cache_plugin插件插件需要的配置文件
|
||||
|
||||
**2、ms_cache的application配置，内容格式如下**
|
||||
{ms_cache, [
|
||||
{data_node, undefined}, % 缓存节点，undefined为缓存app启动在本节点
|
||||
{cache_db_type, ets}, % 缓存在内存的储存位置 proc | ets | mnesia(mnesia暂不支持)
|
||||
{rw_mode, concurrency}, % 缓存读写模式 concurrency-并行读写（脏读脏写） serial-串行读写（经过缓存进程进行读写）
|
||||
{data_sync_mode, key}, % 缓存数据同步模式 key-同步主键（写库时需要根据key读取缓存表的数据，然后同步到数据库） data-同步数据
|
||||
{data_fmt, record}, % 缓存数据存取的格式， record | map， 默认为record
|
||||
{table_worker_count, 8}, % 每个内存表对应处理的进程数（哈希分组到各个进程）
|
||||
{db_save_time, 120000}, % （定时同步参数）缓存定期同步到数据库的时间间隔，单位毫秒；默认值 120000
|
||||
{db_save_count, 1000}, % （定量同步参数）缓存待同步数量上限，达到数量触发同步数据库操作；默认值 1000
|
||||
{db_driver, 'mysql-otp'}, % 使用的数据库驱动（当前只支持mysql-otp第三方驱动，可扩展支持mongo）
|
||||
{db_user, "xrxdzadmin"}, % 数据库用户
|
||||
{db_pass, "xrxdz@4399##"}, % 数据库密码
|
||||
{db_host, "172.16.12.111"}, % 数据库地址
|
||||
{db_port, 3306}, % 数据库端口
|
||||
{db_name, "xrxdz_test_game"}, % 数据库名
|
||||
{db_encoding, utf8}, % 数据库编码（mysql-otp驱动下没用到）
|
||||
{db_select_pool_name, select_pool_name}, % 数据库查询链接名
|
||||
{db_select_pool_size, 32}, % 数据库查询链接池初始容量
|
||||
{db_select_pool_overflow, 128}, % 数据库查询链接池可扩展容量
|
||||
{db_update_pool_name, update_pool_name}, % 数据库更新链接名（增删改都使用该链接）
|
||||
{db_update_pool_size, 10}, % 数据库更新链接池初始容量
|
||||
{db_update_pool_overflow, 0}, % 数据库更新链接池可扩展容量
|
||||
%% 启动前修复数据，通过启动一个erlang节点来处理
|
||||
{db_sync_options, #{
|
||||
create_table => true, % 是否自动创建表
|
||||
alter_table => true, % 是否自动修改表字段(修改主键的话删表重建)
|
||||
patch_run_mod => auto, % 补丁运行模式,auto-启动服务器时自动运行, {env,"ERLANG_RUN_PATCH"}-启动时系统设置了这个环境变量就运行
|
||||
patch_prefix => "patch_", % 补丁文件的前缀名
|
||||
limit => 7 % 最多检查近x条补丁,如有遗漏追加执行,外网建议设置为1
|
||||
}}
|
||||
]}
|
||||
|
||||
**3、配置文件，.table结尾，内容格式如下**
|
||||
|
||||
%% ====================================================================================================
|
||||
%% 表配置说明
|
||||
%%
|
||||
%% {表名, 字段名列表, 主键字段列表, [可选属性map结构]}
|
||||
%%
|
||||
%% {TabName, FieldList, PrimKeyList, [OptionMap]}
|
||||
%%
|
||||
%% TabName :: 数据库对应的表名，也是代码里对应的record名
|
||||
%%
|
||||
%% FieldList :: [Field] 字段列表，对应mysql数据库表的字段
|
||||
%% Field :: {Name, Type, Len, Common, Sync}
|
||||
%% Name :: 字段名
|
||||
%% Type :: 类型，int | binary | list | map | atom | boolean | tuple | any(上面所有类型都可以，数据库用blob存储)
|
||||
%% Len :: 数据库表中该字段的长度，(Len=0，Type= list | map | tuple, 则表示该字段数据库表用text来存储, binary和any则用blob来存储）
|
||||
%% Common :: 字段的说明
|
||||
%% Sync :: 是否同步到数据库表，true | false ，(当Sync=false，表示该字段是临时数据，数据库表也没有该字段)
|
||||
%%
|
||||
%% PrimKeyList :: 表主键，为空默认表的第一个字段作为主键；多个字段表示联合主键
|
||||
%%
|
||||
%% OptionMap :: 可选属性map结构，不配置则默认为#{}；key对应的首个value为默认值，不存在key则为该默认值
|
||||
%% #{
|
||||
%% type => set | ordered_set, % 表类型，目前只支持set和ordered_set，参考ets用法
|
||||
%% indexes => [], % 表索引，自动创建数据库表时自动创建索引
|
||||
%% uid_field => undefined, % 唯一id字段，类似于唯一索引，undefined表示该表不需要唯一id字段（该字段主要是用来生成select max取表的最大唯一id值）
|
||||
%% pre_load => false | true, % 启动时是否全量加载数据
|
||||
%% clear_all => false | true, % 表示可否通过ms_cache:delete_all_objects方法删除表的所有数据(慎用，不重要数据可以用该字段)
|
||||
%% db_save_time => 0 | N |... % （定时同步参数）缓存定期同步到数据库的时间间隔，单位毫秒；（默认读sys.config中ms_cache配置的db_save_time）
|
||||
%% db_save_count => 0 | N |... % （定量同步参数）缓存待同步数量上限，达到数量触发同步数据库操作；（默认读sys.config中ms_cache配置的db_save_count）
|
||||
%% merge_type => 0 | 1 | 2 | 3 % 合服类型,用于生成合服sql，0-保留主从服数据(默认值) 1-保留主服数据 2-保留从服数据 3-删除主从服数据, 4-不处理（合服后处理）
|
||||
%% }.
|
||||
%% ====================================================================================================
|
||||
%% 测试表1, 单主键, 预加载
|
||||
{im_test_1,
|
||||
[
|
||||
{im_long, int, 20, "long type", true},
|
||||
{im_shot, int, 4, "shot type", true},
|
||||
{im_int, int, 11, "int type", true},
|
||||
{im_atom, int, 20, "atom type", true},
|
||||
{im_boolean, boolean, 8, "boolean type", true},
|
||||
{im_binary, binary, 64, "binary type", true},
|
||||
{im_tuple, tuple, 64, "tuple varchar type", true},
|
||||
{im_list, list, 64, "list varchar type", true},
|
||||
{im_map, map, 64, "map varchar type", true},
|
||||
{im_tuple_text, tuple, 0, "tuple text type", true},
|
||||
{im_list_text, list, 0, "list text type", true},
|
||||
{im_map_text, map, 0, "map text type", true},
|
||||
{im_any, any, 0, "any type db-type is blob", true},
|
||||
{im_index, int, 20, "index for table", true}
|
||||
],
|
||||
[im_long], #{ indexes=>[im_db_index], pre_load=>true }
|
||||
}.
|
||||
|
||||
%% 测试表2, 双主键（im_long + im_short）
|
||||
{im_test_2,
|
||||
[
|
||||
{im_long, int, 20, "long type", true},
|
||||
{im_shot, int, 4, "shot type", true},
|
||||
{im_int, int, 11, "int type", true},
|
||||
{im_atom, int, 20, "atom type", true},
|
||||
{im_boolean, boolean, 8, "boolean type", true},
|
||||
{im_binary, binary, 64, "binary type", true},
|
||||
{im_tuple, tuple, 64, "tuple varchar type", true},
|
||||
{im_list, list, 64, "list varchar type", true},
|
||||
{im_map, map, 64, "map varchar type", true},
|
||||
{im_tuple_text, tuple, 0, "tuple text type", true},
|
||||
{im_list_text, list, 0, "list text type", true},
|
||||
{im_map_text, map, 0, "map text type", true},
|
||||
{im_any, any, 0, "any type db-type is blob", true},
|
||||
{im_compound_index1,int, 20, "compound index for table", true},
|
||||
{im_compound_index2,int, 20, "compound index for table", true}
|
||||
],
|
||||
[im_long, im_shot], #{ indexes=>[{im_compound_index1, im_compound_index2}] }
|
||||
}.
|
||||
|
||||
%% 其他说明
|
||||
多key情况下，key的使用形式是tuple；所以不管是单key的字段，还是多key的每个字段，格式都不能为tuple
|
||||
index的创建，多个index用逗号隔开；复合索引用tuple表示
|
||||
|
||||
**4、rebar3_cache_plugin插件及ms_cache库在rebar.config中配置如下**
|
||||
|
||||
%% ms_cache做为第三方依赖库
|
||||
{deps, [
|
||||
mysql
|
||||
, mysql_poolboy
|
||||
, poolboy
|
||||
, ms_cache
|
||||
]}.
|
||||
|
||||
%% 缓存插件定义
|
||||
{plugins, [
|
||||
{rebar3_cache_plugin, "0.1.0"} %% 缓存插件
|
||||
]}.
|
||||
|
||||
%% cache缓存插件配置（支持配置多个app的配置）
|
||||
{cache_opts, [
|
||||
{game_server, [ % app名字
|
||||
{i, "cache"}, % 缓存配置目录
|
||||
{module_name_prefix, ""}, % 文件名前缀，""表示没有前缀名字
|
||||
{module_name_suffix, "_c"}, % 文件名后缀，""表示没有后缀名字
|
||||
{o_erl, "src/auto_gen/cache"}, % erl文件的输出目录
|
||||
{o_hrl, "include/auto_gen/cache"}, % hrl文件的输出目录
|
||||
{o_sql, "sql"}, % sql文件输出目录
|
||||
{strings_as_binaries, true},
|
||||
type_specs
|
||||
]}
|
||||
]}.
|
||||
|
||||
%% 使用命令如下，也可以配置在rebar.config自动执行
|
||||
rebar3 cache clean
|
||||
rebar3 cache compile
|
||||
|
||||
**5、常用方法调用**
|
||||
|
||||
节点内调用 | 跨节点调用
|
||||
ms_cache:lookup(Table, Key) | ms_cache:lookup(Node, Table, Key)
|
||||
ms_cache:lookup_by_key_index(Table, KeyIndex) | ms_cache:lookup_by_key_index(Node, Table, KeyIndex)
|
||||
ms_cache:update(Table, RecOrMap) | ms_cache:update(Node, Table, RecOrMap)
|
||||
ms_cache:delete(Table, Key) | ms_cache:update(Node, Table, Key)
|
||||
ms_cache:insert(Table, RecOrMap) | ms_cache:update(Node, Table, RecOrMap)
|
||||
|
||||
以下是一些常用的仅查询缓存数据的方法（忽略数据库里的数据，通常是一些预先将所有数据加载导内存的表才会用到这些功能）
|
||||
ms_cache:index_read(Table, Key, Pos) | ms_cache:index_read(Node, Table, Key, Pos)
|
||||
ms_cache:size(Table) | ms_cache:size(Node, Table)
|
||||
ms_cache:tab2list(Table) | ms_cache:tab2list(Node, Table)
|
||||
ms_cache:tab2list_foldl(Table, Fun, Acc) | ms_cache:tab2list_foldl(Node, Table, Fun, Acc)
|
||||
|
||||
以下是一些常用的仅查询数据库的方法（忽略缓存里的数据，通常是启服初始化的时候从数据库里获取数据，比如一次性读出角色表的多有名字写入ets用于业务唯一性检测）
|
||||
ms_cache:select_count(Table) | ms_cache:select_count(Node, Table)
|
||||
ms_cache:select_max(Table) | ms_cache:select_max(Node, Table)
|
||||
ms_cache:select_max(Table) | ms_cache:select_max(Node, Table)
|
||||
ms_cache:select_foldl(Table, Fun, Acc) | ms_cache:select_foldl(Node, Table, Fun, Acc)
|
||||
|
||||
ms_cache:lookup_by_key_index/2方法的使用说明
|
||||
在多key表的情况下，按照第一个key获取对应的所有数据
|
||||
在单key表的情况下，等同于 [ms_cache:lookup/2],注意返回值是个列表
|
||||
|
||||
**6、其他说明**
|
||||
|
||||
文件中定义了数据的record结构，并提供了record转map的功能；
|
||||
record生成时自动增加'_id'作为record的key
|
||||
map格式使用时，数据必需带上'_table'作为区分表的key，map该key对应的value为表名atom
|
||||
'_id' 和 '_table' 作为保留字段，在配置中不能作为数据的字段，为了便于阅读，id字段最好也别用
|
||||
|
||||
ms_cache库中包含 ms_cache 和 ms_cache_internal 模块
|
||||
早期只有ms_cache_internal，和其他模块一起组成一个独立运行的缓存应用系统；（所以ms_cache提供的方法这里基本也有）
|
||||
ms_cache是后期为了简化部分业务逻辑，提高读写性能引入的，所以ms_cache有些接口是在业务的需求下定义的，并没有那么纯粹
|
||||
ms_cache主要做的事情
|
||||
1、通过ms_cache:init(Table, Owner, Args),指定进程可以操作Table表的哪些数据，我们也将该进程称为这些数据的宿主进程
|
||||
Owner = any，表示可以操作Table表的所有数据,这里有个特殊的情况，表的key字段不能为原子any
|
||||
Owner = KeyOrIndex, 对于单key表，只能操作KeyOrIndex为主键的一条数据；对于多key表，可以操作第1个key未KeyOrIndex的多条数据
|
||||
2、Owner /= any进程字典内可以指定存储一份ms_cache:init指定的可以操作的数据；并通过Args参数，可以自行决定调用方法将数据写入缓存的时机，缓存的数据能否被手动移除
|
||||
3、宿主进程可以实时访问最新的数据，非宿主进程只能访问缓存ets的数据，是否需要最新取决于Args参数的设定
|
||||
@@ -0,0 +1,204 @@
|
||||
-ifndef(__MS_CACHE_HRL__).
|
||||
-define(__MS_CACHE_HRL__, true).
|
||||
|
||||
-include("base.hrl").
|
||||
|
||||
-define(MS_CACHE, ms_cache).
|
||||
|
||||
-define(WORKER_COUNTER_DEFAULT, 8).
|
||||
-define(WORKER_INIT_TIME_MIN, 100).
|
||||
|
||||
%% 需要更新的数据超过指定数量，立即触发存盘，默认只为1000
|
||||
-define(SYNC_NUM, ms_cache_config:get_env(db_save_count, 1000)).
|
||||
|
||||
%% 缓存保存到数据库时间，默认10分钟保存一次
|
||||
-define(DB_SAVE_TIME, ms_cache_config:get_env(db_save_time, 600000)).
|
||||
|
||||
%% 安全模式（待修改的数据有备份到dets或其他落地数据库）
|
||||
-define(SAFE_MODE, ms_cache_config:get_env(safe_mode, false)).
|
||||
|
||||
%% 错误信息
|
||||
-define(ERR_MSG(Tab, Key, Reason), ms_cache_util:get_error_message(Tab, Key, Reason)).
|
||||
|
||||
%% 根据类型获取缓存表名
|
||||
-define(TABLE_NAME(Tab, Type), (ms_cache_util:get_table_name(Tab, Type))).
|
||||
-define(TABLE_NAME(Tab, Type, WorkNum), (ms_cache_util:get_table_name(Tab, Type, WorkNum))).
|
||||
|
||||
%% 获取缓存数据节点
|
||||
-define(DATA_NODE, (ms_cache_util:get_data_node())).
|
||||
|
||||
%% 获取缓存表处理的模块
|
||||
%%-define(TABLE_HANDLER(Tab), (list_to_atom(lists:concat([Tab, "_c"])))).
|
||||
-define(TABLE_HANDLER(Tab), (ms_cache_config:get({?MS_CACHE, handler, Tab}))).
|
||||
-define(SET_TABLE_HANDLER(Tab, Handler), (ms_cache_config:put({?MS_CACHE, handler, Tab}, Handler))).
|
||||
|
||||
%% 缓存数据转换为record形式
|
||||
-define(TO_REC(Data), (ms_cache_util:map_to_record(Data))).
|
||||
|
||||
%% 缓存数据转换为map形式
|
||||
-define(TO_MAP(Data), (ms_cache_util:record_to_map(Data))).
|
||||
|
||||
%% 缓存数据格式转换
|
||||
-define(FMT_DATA(Data), (ms_cache_util:format_data(Data))).
|
||||
|
||||
%% 单次清理缓存数据的数量
|
||||
-define(CACHE_REMOVE_LIMIT, 1000).
|
||||
|
||||
%% 获取数据库连接池名
|
||||
-define(SELECT_POOL, ms_cache_config:get_env(db_select_pool_name, undefined)).
|
||||
-define(UPDATE_POOL, ms_cache_config:get_env(db_update_pool_name, undefined)).
|
||||
|
||||
%% 读写模式 concurrency | serial
|
||||
-define(CONCURRENCY, concurrency).
|
||||
-define(SERIAL, serial).
|
||||
-define(RW_MODE, ms_cache_config:get_env(rw_mode, ?CONCURRENCY)).
|
||||
|
||||
%% 数据同步模式 data | key todo key模式当前有bug，关服时缓存data表先被删除，导致同步进程查询的时候报错
|
||||
-define(DATA_SYNC_MODE_DATA, data). %% 同步变化的整个数据 record | map
|
||||
-define(DATA_SYNC_MODE_KEY, key). %% 只同步变化的key
|
||||
-define(DATA_SYNC_MODE, ms_cache_config:get_env(data_sync_mode, ?DATA_SYNC_MODE_DATA)).
|
||||
|
||||
%% 获取表的handler名称
|
||||
-define(TABLE_HANDLER_NAME(Table), list_to_atom(lists:concat([Table, "_c"]))).
|
||||
|
||||
-define(MS_CACHE_DB_DROP_AUTH, ms_cache_config:get_env(db_drop_auth, ?FALSE)).
|
||||
|
||||
-define(OPT_TABLE_MAX, 10).
|
||||
-define(OPT_TABLE_LIST(Table), [erlang:list_to_atom(erlang:atom_to_list(?TABLE_NAME(Table, opt)) ++ "_" ++ erlang:integer_to_list(Num)) || Num<-lists:seq(1, ?OPT_TABLE_MAX)]).
|
||||
|
||||
|
||||
%% 缓存使用的内存表.
|
||||
%% ms_cache_mdb : mnesia作为缓存表
|
||||
%% ms_cache_edb : ets set数据表 + ets ordered_set索引表 （仿mnesia写法，mnesia简化版，综合性能最好）
|
||||
%% ms_cache_edb2 : ets set数据表 + ets bag索引表 (插入和删除性能较差)
|
||||
%% ms_cache_edb3 : ets set数据表 + ets set索引表 （大数据性能差，别用，只用来测试对比）
|
||||
-define(MS_CACHE_DB_MOD, (ms_cache_util:get_cache_db_mod())).
|
||||
|
||||
%% 数据库操作标志
|
||||
-record(ms_cache_opt, {
|
||||
key = 0,
|
||||
opt = none, %% insert | update | delete
|
||||
opt_rec = none
|
||||
}).
|
||||
|
||||
%% 复合主键的数据，以第一个key作为索引的所有key列表
|
||||
-record(ms_cache_idx, {
|
||||
key = 0,
|
||||
val = []
|
||||
}).
|
||||
|
||||
%% 数据加载标志
|
||||
-record(ms_cache_load, {
|
||||
key = 0,
|
||||
val = none %%{Key, none} - 无Key对应数据 | {Index, index_loaded} - 复合主键以第一个key为所以的数据已加载 | {Table, table_loaded} - 整表数据已加载
|
||||
}).
|
||||
|
||||
-record(ms_cache_expired, {
|
||||
key = 0, %% 缓存清理key | 缓存清理index（复合主键）
|
||||
val = none
|
||||
}).
|
||||
|
||||
|
||||
|
||||
|
||||
%% atom define
|
||||
-ifndef(OK).
|
||||
-define(OK, ok).
|
||||
-endif.
|
||||
-ifndef(TRUE).
|
||||
-define(TRUE, true).
|
||||
-endif.
|
||||
-ifndef(FALSE).
|
||||
-define(FALSE, false).
|
||||
-endif.
|
||||
-ifndef(UNDEFINED).
|
||||
-define(UNDEFINED, undefined).
|
||||
-endif.
|
||||
-ifndef(INDEFINITELY).
|
||||
-define(INDEFINITELY, indefinitely).
|
||||
-endif.
|
||||
-ifndef(CONTINUE).
|
||||
-define(CONTINUE, continue).
|
||||
-endif.
|
||||
-ifndef(IGNORE).
|
||||
-define(IGNORE, ignore).
|
||||
-endif.
|
||||
-ifndef(INFINITY).
|
||||
-define(INFINITY, infinity).
|
||||
-endif.
|
||||
-ifndef(ERROR).
|
||||
-define(ERROR, error).
|
||||
-endif.
|
||||
-ifndef(NONE).
|
||||
-define(NONE, none).
|
||||
-endif.
|
||||
-ifndef(STOP).
|
||||
-define(STOP, stop).
|
||||
-endif.
|
||||
-ifndef(SKIP).
|
||||
-define(SKIP, skip).
|
||||
-endif.
|
||||
|
||||
%% 封装处理handle_continue
|
||||
-define(TRY_CONTINUE(Msg, State),
|
||||
try
|
||||
do_continue(Msg, State)
|
||||
catch _:ErrMsgDoNotRepeatName:Trace ->
|
||||
?ERROR(?MODULE, "handle continue: Msg = ~p, ErrMsg = ~p, Trace = ~p", [Msg, ErrMsgDoNotRepeatName, Trace]),
|
||||
{noreply, State}
|
||||
end
|
||||
).
|
||||
%% 封装处理handle_call
|
||||
-define(TRY_CALL(Msg, From, State),
|
||||
try
|
||||
do_call(Msg, From, State)
|
||||
catch _:ErrMsgDoNotRepeatName:Trace ->
|
||||
?ERROR(?MODULE, "handle call: Msg = ~p, ErrMsg = ~p, Trace = ~p", [Msg, ErrMsgDoNotRepeatName, Trace]),
|
||||
{reply, ok, State}
|
||||
end
|
||||
).
|
||||
-define(TRY_CALL(Msg, State),
|
||||
try
|
||||
do_call(Msg, State)
|
||||
catch _:ErrMsgDoNotRepeatName:Trace ->
|
||||
?ERROR(?MODULE, "handle call: Msg = ~p, ErrMsg = ~p, Trace = ~p", [Msg, ErrMsgDoNotRepeatName, Trace]),
|
||||
{reply, ok, State}
|
||||
end
|
||||
).
|
||||
%% 封装处理handle_cast
|
||||
-define(TRY_CAST(Msg, State),
|
||||
try
|
||||
do_cast(Msg, State)
|
||||
catch _:ErrMsgDoNotRepeatName:Trace ->
|
||||
?ERROR(?MODULE, "handle cast: Msg = ~p, ErrMsg = ~p, Trace = ~p", [Msg, ErrMsgDoNotRepeatName, Trace]),
|
||||
{noreply, State}
|
||||
end
|
||||
).
|
||||
%% 封装处理handle_info
|
||||
-define(TRY_INFO(Msg, State),
|
||||
try
|
||||
do_info(Msg, State)
|
||||
catch _:ErrMsgDoNotRepeatName:Trace ->
|
||||
?ERROR(?MODULE, "handle info: Msg = ~p, ErrMsg = ~p, Trace = ~p", [Msg, ErrMsgDoNotRepeatName, Trace]),
|
||||
{noreply, State}
|
||||
end
|
||||
).
|
||||
|
||||
|
||||
%%% 打印函数
|
||||
|
||||
%% 进程字典的读取和存储
|
||||
-define(PUT_TO_PROC_DICT(Key, Val), erlang:put({?MODULE, Key}, Val)).
|
||||
-define(GET_FROM_PROC_DICT(Key), erlang:get({?MODULE, Key})).
|
||||
-define(GET_FROM_PROC_DICT(Key, Default),
|
||||
case erlang:get({?MODULE, Key}) of
|
||||
?UNDEFINED ->
|
||||
Default;
|
||||
Val ->
|
||||
Val
|
||||
end ).
|
||||
-define(ERASE_FROM_PROC_DICT(Key), erlang:erase({?MODULE, Key})).
|
||||
|
||||
-type ms_cache_key() :: integer() | atom() | binary().
|
||||
|
||||
-endif.
|
||||
@@ -0,0 +1,17 @@
|
||||
{application, ms_cache,
|
||||
[{description, "An OTP application"},
|
||||
{vsn, "0.1.0"},
|
||||
{registered, []},
|
||||
{mod, {ms_cache_app, []}},
|
||||
{applications,
|
||||
[kernel,
|
||||
stdlib,
|
||||
mysql_poolboy,
|
||||
mnesia
|
||||
]},
|
||||
{env,[]},
|
||||
{modules, []},
|
||||
|
||||
{licenses, ["Apache 2.0"]},
|
||||
{links, []}
|
||||
]}.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
%%% -------------------------------------------------------------------
|
||||
%%% Author : wangyalun
|
||||
%%% Description : 缓存application
|
||||
%%% Created : 2019-01-14
|
||||
%%% -------------------------------------------------------------------
|
||||
-module(ms_cache_app).
|
||||
|
||||
-include("base.hrl").
|
||||
-behaviour(application).
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
%% Application callbacks
|
||||
-export([start/2, stop/1]).
|
||||
-export([start_all/0]).
|
||||
|
||||
%% ===================================================================
|
||||
%% Application callbacks
|
||||
%% ===================================================================
|
||||
|
||||
start(_StartType, _StartArgs) ->
|
||||
case ms_cache_sup:start_link() of
|
||||
{ok, Sup} ->
|
||||
case ms_cache_util:get_cache_db_mod() of
|
||||
ms_cache_mdb ->
|
||||
case mnesia:system_info(use_dir) of
|
||||
true ->
|
||||
mnesia:start();
|
||||
false ->
|
||||
mnesia:stop(),
|
||||
mnesia:create_schema([erlang:node()]),
|
||||
mnesia:start()
|
||||
end,
|
||||
%% 这句语句，会使所有mnesia的ram_copies表结构不存储在本地磁盘；不加的话上次关服创建的表下次启服时表结构是存在的
|
||||
mnesia:change_table_copy_type(schema, node(), ram_copies),
|
||||
|
||||
%% 同步表使用dets存储，不用mnesia的愿意如下
|
||||
%% 其他表（数据表、加载表和过期表使用mnesia临时表）
|
||||
%% 这里的原因是mnesia:change_table_copy_type(schema, node(), ram_copies)之后建的都是内存临时表
|
||||
%% 而同步表，在数据库连不上 /节点挂掉的情况，需要存储，减少数据的丢失，所以不能用mnesia临时表
|
||||
%% 如果mnesia不指定为临时表，则其他表的表结构会存在mnesia，如果结构变动启动服务器非常慢
|
||||
?IF(?SAFE_MODE, filelib:ensure_dir("dets/*"));
|
||||
_ ->
|
||||
?SKIP
|
||||
end,
|
||||
{ok, Sup};
|
||||
Error ->
|
||||
?ERROR(?MODULE, "application start error : ~p", [Error]),
|
||||
Error
|
||||
end.
|
||||
|
||||
stop(_State) ->
|
||||
?INFO(?MODULE, "stop ms_cache application", []),
|
||||
ok.
|
||||
|
||||
%% rebar3热更会用版本内的sys.config的配置替换运行的代码，导致热更的application的envs被替换
|
||||
%% 游戏各个服配置不一样，难以在打升级包的时候就把sys.config定义好
|
||||
%% 通常情况热更的时候，这些配置就会被替换成空失效
|
||||
%% config_change是热更时替换sys.config数据后调用被热更配置的app，所以在这里将app的ets回写到ac_tab
|
||||
%% 注意：不能用application:set_env来处理，call会导致死锁，因为这个处理是在application_controller进程执行
|
||||
%%config_change(_Changed, _New, _Removed) ->
|
||||
%% %% 统一由启动cache的app去修改app的环境变量
|
||||
%% ?INFO(?MODULE, "ms_cache hotfix config change", []),
|
||||
%% ok.
|
||||
|
||||
start_all() ->
|
||||
application:start(sasl),
|
||||
application:start(crypto),
|
||||
application:start(inets),
|
||||
application:start(asn1),
|
||||
application:start(public_key),
|
||||
application:start(ssl),
|
||||
application:start(goldrush),
|
||||
application:start(mysql_poolboy),
|
||||
application:start(ms_cache),
|
||||
ok.
|
||||
@@ -0,0 +1,50 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2025, <COMPANY>
|
||||
%%% @doc
|
||||
%%%
|
||||
%%% @end
|
||||
%%% Created : 10. 1月 2025 11:39
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_config).
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% API
|
||||
-export([
|
||||
get_env/2
|
||||
, put/2
|
||||
, get/1
|
||||
, get/2
|
||||
]).
|
||||
|
||||
%% 获取application的环境变量，application:get_env/2
|
||||
get_env(Key, Def) ->
|
||||
case persistent_term:get({?MODULE, env, Key}, {?UNDEFINED, ?NONE}) of
|
||||
{?UNDEFINED, ?NONE} ->
|
||||
case application:get_env(?MS_CACHE, Key) of
|
||||
?UNDEFINED ->
|
||||
Def;
|
||||
{?OK, Value} ->
|
||||
persistent_term:put({?MODULE, env, Key}, Value),
|
||||
Value
|
||||
end;
|
||||
Value ->
|
||||
Value
|
||||
end.
|
||||
%% application:get_env(?MS_CACHE, Key, Def).
|
||||
|
||||
%% 设置常量
|
||||
put(Key, Value) ->
|
||||
case persistent_term:get({?MODULE, const, Key}, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
persistent_term:put({?MODULE, const, Key}, Value),
|
||||
?OK;
|
||||
Value ->
|
||||
error({error_key_exist, Key, Value})
|
||||
end.
|
||||
|
||||
%% 获取设置的常量
|
||||
get(Key) ->
|
||||
get(Key, ?UNDEFINED).
|
||||
|
||||
get(Key, Def) ->
|
||||
persistent_term:get({?MODULE, const, Key}, Def).
|
||||
@@ -0,0 +1,395 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2019, <COMPANY>
|
||||
%%% @doc
|
||||
%%% ms_cache与数据库交互的进程
|
||||
%%% @end
|
||||
%%% Created : 2019-09-01 00:00:00
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_dao_merge_gs).
|
||||
-include("base.hrl").
|
||||
-behaviour(gen_server).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% include
|
||||
%%--------------------------------------------------------------------
|
||||
-include("ms_cache.hrl").
|
||||
%% Auto recovered record definitions extracted from decompiled Erlang source.
|
||||
%% Keep record definitions in headers; do not define them directly in .erl files.
|
||||
|
||||
-record(state, {}).
|
||||
%%--------------------------------------------------------------------
|
||||
%% API exports
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-export([
|
||||
start_link/2
|
||||
, set_opt/5 %% 设置数据操作标志
|
||||
, hand_sync/1 %% 手动同步数据到数据库
|
||||
, hand_sync/2 %% 手动同步指定数据到数据库
|
||||
, set_remove_cache/2 %% 设置移除缓存
|
||||
, del_remove_cache/2 %% 取消移除缓存
|
||||
, sync_finish/2 %% dao进程通知同步完毕
|
||||
%% , create_table/2
|
||||
]).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% gen_server callbacks
|
||||
%%--------------------------------------------------------------------
|
||||
-export([
|
||||
init/1
|
||||
,handle_continue/2
|
||||
,handle_call/3
|
||||
,handle_cast/2
|
||||
,handle_info/2
|
||||
,terminate/2
|
||||
,code_change/3
|
||||
]).
|
||||
|
||||
%% 缓存数据表与数据库交互的进程
|
||||
-define(SERVER(Table), list_to_atom(atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Table))).
|
||||
|
||||
|
||||
%%%===================================================================
|
||||
%%% API
|
||||
%%%===================================================================
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% Starts the server
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(start_link(Table :: atom(), SafeMode :: boolean()) ->
|
||||
{ok, Pid :: pid()} | ignore | {error, Reason :: term()}).
|
||||
start_link(Table, SafeMode) ->
|
||||
gen_server:start_link({local, ?SERVER(Table)}, ?MODULE, [Table, SafeMode], []).
|
||||
|
||||
%%%===================================================================
|
||||
%%% gen_server callbacks
|
||||
%%%===================================================================
|
||||
%% 设置数据操作标志
|
||||
set_opt(Table, Key, Otp, Record, Flush) ->
|
||||
SyncData = ?IF(?DATA_SYNC_MODE == ?DATA_SYNC_MODE_DATA, Record, ?NONE),
|
||||
gen_server:cast(?SERVER(Table), {set_opt, Key, Otp, SyncData, Flush}).
|
||||
|
||||
hand_sync(Table) ->
|
||||
erlang:send(?SERVER(Table), hand_sync, [noconnect]).
|
||||
|
||||
hand_sync(Table, KeyOrKeys) ->
|
||||
erlang:send(?SERVER(Table), {hand_sync, KeyOrKeys}, [noconnect]).
|
||||
|
||||
set_remove_cache(Table, Index) ->
|
||||
gen_server:cast(?SERVER(Table), {set_remove_cache, Index, erlang:self()}).
|
||||
|
||||
del_remove_cache(Table, Index) ->
|
||||
gen_server:cast(?SERVER(Table), {del_remove_cache, Index, erlang:self()}).
|
||||
|
||||
sync_finish(Table, OptTable) ->
|
||||
gen_server:cast(?SERVER(Table), {sync_finish, OptTable}).
|
||||
|
||||
%%create_table(Table, SafeMode) ->
|
||||
%% {?OK, State} = get_default_state(Table, SafeMode),
|
||||
%%%% ms_cache_dao_merge_gs_lib:create_table(State),
|
||||
%% ?OK.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Initializes the server
|
||||
%%
|
||||
%% @spec init(Args) -> {ok, State} |
|
||||
%% {ok, State, Timeout} |
|
||||
%% ignore |
|
||||
%% {stop, Reason}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(init(Args :: term()) ->
|
||||
{ok, State :: #state{}} | {ok, State :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term()} | ignore).
|
||||
|
||||
init([Table, SafeMode]) ->
|
||||
%% 首次存档时间
|
||||
FirstTime = ?WORKER_INIT_TIME_MIN + rand:uniform(?DB_SAVE_TIME),
|
||||
%% ?DEBUG(?MODULE, "~p init", [?SERVER(Table)]),
|
||||
erlang:process_flag(trap_exit, true),
|
||||
{?OK, State} = get_default_state(Table, SafeMode),
|
||||
DbSaveTime = maps:get(db_save_time, State),
|
||||
TimerRef = erlang:start_timer(DbSaveTime + FirstTime, self(), sync),
|
||||
{ok, State#{timer_ref:=TimerRef}}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling all non call/cast messages
|
||||
%%
|
||||
%% @spec handle_continue(Info, State) -> {noreply, State} |
|
||||
%% {noreply, State, Timeout} |
|
||||
%% {stop, Reason, State}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_continue(Info :: timeout() | term(), State :: #state{}) ->
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_continue(Info, State) ->
|
||||
?TRY_CONTINUE(Info, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling call messages
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()},
|
||||
State :: #state{}) ->
|
||||
{reply, Reply :: term(), NewState :: #state{}} |
|
||||
{reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} |
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), Reply :: term(), NewState :: #state{}} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_call(Request, From, State) ->
|
||||
?TRY_CALL(Request, From, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling cast messages
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_cast(Request :: term(), State :: #state{}) ->
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_cast(Request, State) ->
|
||||
?TRY_CAST(Request, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling all non call/cast messages
|
||||
%%
|
||||
%% @spec handle_info(Info, State) -> {noreply, State} |
|
||||
%% {noreply, State, Timeout} |
|
||||
%% {stop, Reason, State}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_info(Info :: timeout() | term(), State :: #state{}) ->
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_info(Info, State) ->
|
||||
?TRY_INFO(Info, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% This function is called by a gen_server when it is about to
|
||||
%% terminate. It should be the opposite of Module:init/1 and do any
|
||||
%% necessary cleaning up. When it returns, the gen_server terminates
|
||||
%% with Reason. The return value is ignored.
|
||||
%%
|
||||
%% @spec terminate(Reason, State) -> void()
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()),
|
||||
State :: #state{}) -> term()).
|
||||
terminate(Reason, #{ tab:=Table } = State) ->
|
||||
?DEBUG(?MODULE, "~p terminate, Reason = ~p, Table = ~p", [?SERVER(Table), Reason, Table]),
|
||||
ms_cache_dao_merge_gs_lib:do_sync(State, terminate),
|
||||
ok.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Convert process state when code is changed
|
||||
%%
|
||||
%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{},
|
||||
Extra :: term()) ->
|
||||
{ok, NewState :: #state{}} | {error, Reason :: term()}).
|
||||
code_change(_OldVsn, State, _Extra) ->
|
||||
{ok, State}.
|
||||
|
||||
%%%===================================================================
|
||||
%%% gen_server callbacks Internal
|
||||
%%%===================================================================
|
||||
do_continue(check_limit_sync, #{opt_tabs:=[], timer_ref:=?UNDEFINED} = State) ->
|
||||
{noreply, State};
|
||||
do_continue(check_limit_sync, #{opt_tabs:=[], timer_ref:=OldTimerRef} = State) ->
|
||||
NewState =
|
||||
case ms_cache_dao_merge_gs_lib:is_sync_now(State) of
|
||||
?TRUE ->
|
||||
erlang:erase(OldTimerRef),
|
||||
State#{timer_ref:=?UNDEFINED};
|
||||
?FALSE ->
|
||||
State
|
||||
end,
|
||||
{noreply, NewState};
|
||||
do_continue(check_limit_sync, #{opt_tabs:=[H | T], timer_ref := OldTimerRef, db_save_time := DbSaveTime} = State) ->
|
||||
NewState =
|
||||
case ms_cache_dao_merge_gs_lib:is_sync_now(State) of
|
||||
?TRUE ->
|
||||
erlang:erase(OldTimerRef),
|
||||
TimerRef = erlang:start_timer(DbSaveTime, self(), sync),
|
||||
ms_cache_dao_merge_gs_lib:do_sync(State, limit_sync),
|
||||
%% ?INFO(?MODULE, "change OptTable, ~p => ~p", [maps:get(opt_tab, State), H]),
|
||||
State#{ opt_tab:=H, opt_tabs:=T, timer_ref:=TimerRef };
|
||||
?FALSE ->
|
||||
State
|
||||
end,
|
||||
{noreply, NewState}.
|
||||
|
||||
do_call(_Request, _From, State) ->
|
||||
{reply, ok, State}.
|
||||
|
||||
do_cast({set_opt, _, delete_all_objects, _, _}, #{ delete_all_objects:=?FALSE } = State) ->
|
||||
%% 删除所有需要待修改的数据
|
||||
ms_cache_dao_merge_gs_lib:delete_all_objects(State),
|
||||
{noreply, State#{ delete_all_objects:=?TRUE }};
|
||||
do_cast({set_opt, _, delete_all_objects, _, _}, #{ delete_all_objects:=?TRUE } = State) -> %% 数据已被删除
|
||||
{noreply, State};
|
||||
|
||||
do_cast({set_opt, Key, insert, Record, Flush}, State) ->
|
||||
case ms_cache_dao_merge_gs_lib:lookup(State, Key) of
|
||||
[] ->
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=insert,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=delete }] -> %% 如果还没有被删除，变为更新
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=update }] -> %% 有可能truncate表后（truncate临界区update），重新插入
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=insert,opt_rec = OldRecord }] -> %% 重复插入以最新的数据为准（兼容写法）
|
||||
?ERROR(?MODULE, "inserted when set opt insert, OldRec = ~p, NewRec = ~p", [OldRecord, Record]),
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=insert,opt_rec=Record });
|
||||
Other -> %% 未知情况，修改为插入（兼容写法）
|
||||
?ERROR(?MODULE, "other opt seted when set opt insert, Other = ~p, NewRec = ~p", [Other, Record]),
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=insert,opt_rec=Record })
|
||||
end,
|
||||
?IF(Flush, ms_cache_dao_merge_gs_lib:try_sync_one_to_db(State, Key)),
|
||||
{noreply, State#{ delete_all_objects:=?FALSE }, {continue, check_limit_sync}}; %% 设置数据未被全部删除
|
||||
do_cast({set_opt, Key, update, Record, Flush}, State) ->
|
||||
case ms_cache_dao_merge_gs_lib:lookup(State, Key) of
|
||||
[] ->
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=insert }] ->
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=insert,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=update }] ->
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record });
|
||||
[#ms_cache_opt{ opt=delete }] -> %% 这种情况应该不存在（兼容写法）
|
||||
?ERROR(?MODULE, "deleted when set opt update, Record = ~p", [Record]),
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record });
|
||||
Other -> %% 未知情况，修改为更新（兼容写法）
|
||||
?ERROR(?MODULE, "other opt seted when set opt update, Other = ~p, NewRec = ~p", [Other, Record]),
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=update,opt_rec=Record })
|
||||
end,
|
||||
?IF(Flush, ms_cache_dao_merge_gs_lib:try_sync_one_to_db(State, Key)),
|
||||
{noreply, State#{ delete_all_objects:=?FALSE }, {continue, check_limit_sync}}; %% 设置数据未被全部删除
|
||||
do_cast({set_opt, Key, delete, _, Flush}, State) ->
|
||||
case ms_cache_dao_merge_gs_lib:lookup(State, Key) of
|
||||
[] -> %% 还没插入直接删掉插入标志即可
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=delete,opt_rec=none });
|
||||
[#ms_cache_opt{ opt=insert }] -> %% 还没插入直接删掉插入标志即可
|
||||
ms_cache_dao_merge_gs_lib:delete(State, Key);
|
||||
[#ms_cache_opt{ opt=update }] -> %% 还没插入直接删掉插入标志即可
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=delete,opt_rec=none });
|
||||
[#ms_cache_opt{ opt=delete }] -> %% 重复删除，兼容写法
|
||||
%% ?ERROR(?MODULE, "deleted when set opt delete (duplicate delete), OptTable~p Key = ~p", [OptTable, Key]),
|
||||
ok;
|
||||
Other -> %% 未知情况，修改为删除（兼容写法）
|
||||
?ERROR(?MODULE, "other opt seted when set opt delete, Other = ~p, Key = ~p", [Other, Key]),
|
||||
ms_cache_dao_merge_gs_lib:insert(State, #ms_cache_opt{ key=Key,opt=delete,opt_rec=none })
|
||||
end,
|
||||
?IF(Flush, ms_cache_dao_merge_gs_lib:try_sync_one_to_db(State, Key)),
|
||||
{noreply, State, {continue, check_limit_sync}};
|
||||
do_cast({set_remove_cache, Index, Pid}, #{opt_tab:=OptTable} = State) ->
|
||||
ms_cache_dao_merge_gs_lib:set_remove_cache(OptTable, Index, Pid),
|
||||
{noreply, State};
|
||||
do_cast({del_remove_cache, Index, _Pid}, #{opt_tab:=OptTable} = State) ->
|
||||
ms_cache_dao_merge_gs_lib:del_remove_cache(OptTable, Index),
|
||||
{noreply, State};
|
||||
%% 手动同步
|
||||
%% TODO 同步在同一个进程数据量大可能会堵消息，先阻塞慢慢消耗消息吧，毕竟这个进程并不影响业务（cup/内存不爆的情况）
|
||||
%%do_cast(hand_sync, State) ->
|
||||
%% ms_cache_dao_merge_gs_lib:do_sync(State, hand_sync),
|
||||
%% {noreply, State};
|
||||
do_cast(hand_sync, #{tab:=Table, opt_tab:=OptTable, opt_tabs := [H | T], timer_ref := OldTimerRef, db_save_time := DbSaveTime} = State) ->
|
||||
erlang:erase(OldTimerRef),
|
||||
TimerRef = erlang:start_timer(DbSaveTime, self(), sync),
|
||||
ms_cache_dao_sync_gs:sync(Table, OptTable, timeout_sync),
|
||||
%% ?INFO(?MODULE, "change OptTable, ~p => ~p", [maps:get(opt_tab, State), H]),
|
||||
{noreply, State#{ opt_tab=>H, opt_tabs=>T, timer_ref=>TimerRef }};
|
||||
do_cast(hand_sync, #{tab:=Table, opt_tabs := [], timer_ref := OldTimerRef} = State) ->
|
||||
?IF(erlang:is_reference(OldTimerRef), erlang:erase(OldTimerRef)),
|
||||
?ERROR(?MODULE, "no new opt_table to use, Table = ~p", [Table]),
|
||||
{noreply, State#{ timer_ref:=?UNDEFINED }};
|
||||
%% 手动同步同步多个数据到数据库
|
||||
do_cast({hand_sync, Keys}, State) when is_list(Keys) ->
|
||||
ms_cache_dao_merge_gs_lib:try_sync_list_to_db(State, Keys),
|
||||
{noreply, State};
|
||||
%% 手动同步同步单一数据到数据库
|
||||
do_cast({hand_sync, Key}, State) ->
|
||||
ms_cache_dao_merge_gs_lib:try_sync_one_to_db(State, Key),
|
||||
{noreply, State};
|
||||
%% 同步完成
|
||||
do_cast({sync_finish, FinishOptTable}, #{tab:=Table, opt_tab := OptTable, opt_tabs := [], timer_ref:=?UNDEFINED, db_save_time := DbSaveTime} = State) ->
|
||||
TimerRef = erlang:start_timer(DbSaveTime, self(), sync),
|
||||
ms_cache_dao_merge_gs_lib:sync_finish(FinishOptTable),
|
||||
ms_cache_dao_sync_gs:sync(Table, OptTable, timeout_sync),
|
||||
{noreply, State#{opt_tab:=FinishOptTable, timer_ref => TimerRef}};
|
||||
do_cast({sync_finish, FinishOptTable}, #{ opt_tabs := OptTables } = State) ->
|
||||
ms_cache_dao_merge_gs_lib:sync_finish(FinishOptTable),
|
||||
{noreply, State#{opt_tabs:=[FinishOptTable | OptTables]}}.
|
||||
|
||||
%% 自动同步
|
||||
do_info({timeout, OldTimerRef, sync}, #{tab:=Table, opt_tab:=OptTable, opt_tabs := [H | T], timer_ref:=OldTimerRef, safe_mode:=SafeMode, db_save_time:=DbSaveTime} = State) ->
|
||||
%% todo 注意这里不能判断OptTable表为空，就走同步数据；这样会导致移除缓存的数据异常
|
||||
TimerRef = erlang:start_timer(DbSaveTime, self(), sync),
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
case DbCacheMod:table_info(OptTable, size) > 0 of
|
||||
?TRUE ->
|
||||
ms_cache_dao_sync_gs:sync(Table, OptTable, timeout_sync),
|
||||
{noreply, State#{ opt_tab=>H, opt_tabs=>T, timer_ref=>TimerRef }};
|
||||
?FALSE ->
|
||||
{noreply, State#{ timer_ref=>TimerRef }}
|
||||
end;
|
||||
do_info({timeout, OldTimerRef, sync}, #{tab:=Table, opt_tabs := [], timer_ref:=OldTimerRef} = State) ->
|
||||
?ERROR(?MODULE, "no new opt_table to use, Table = ~p", [Table]),
|
||||
{noreply, State#{timer_ref=>?UNDEFINED}};
|
||||
do_info({timeout, _TimerRef, sync}, State) ->
|
||||
{noreply, State}.
|
||||
%%%===================================================================
|
||||
%%% Internal functions
|
||||
%%%===================================================================
|
||||
get_default_state(Table, SafeMode) ->
|
||||
[H | T] = ?OPT_TABLE_LIST(Table),
|
||||
Handler = ?TABLE_HANDLER(Table),
|
||||
DbSaveTime =
|
||||
case Handler:db_save_time() of
|
||||
?UNDEFINED ->
|
||||
?DB_SAVE_TIME;
|
||||
DbSaveTime0 ->
|
||||
DbSaveTime0
|
||||
end,
|
||||
DbSaveCount =
|
||||
case Handler:db_save_count() of
|
||||
?UNDEFINED ->
|
||||
?SYNC_NUM;
|
||||
DbSaveCount0 ->
|
||||
DbSaveCount0
|
||||
end,
|
||||
State = #{
|
||||
tab => Table,
|
||||
opt_tab => H,
|
||||
opt_tabs => T,
|
||||
handler => ?TABLE_HANDLER(Table),
|
||||
delete_all_objects => ?FALSE,
|
||||
safe_mode => SafeMode,
|
||||
timer_ref => ?UNDEFINED,
|
||||
db_save_time => DbSaveTime,
|
||||
db_save_count => DbSaveCount
|
||||
},
|
||||
{?OK, State}.
|
||||
@@ -0,0 +1,159 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2023, <COMPANY>
|
||||
%%% @doc
|
||||
%%%
|
||||
%%% @end
|
||||
%%% Created : 03. 7月 2023 20:18
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_dao_merge_gs_lib).
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% API
|
||||
-export([
|
||||
%% create_table/1
|
||||
delete_all_objects/1
|
||||
, lookup/2
|
||||
, insert/2
|
||||
, delete/2
|
||||
, close/1
|
||||
]).
|
||||
|
||||
-export([
|
||||
do_sync/2
|
||||
, is_sync_now/1
|
||||
, get_sync_num/1
|
||||
, try_sync_one_to_db/2
|
||||
, try_sync_list_to_db/2
|
||||
, set_remove_cache/3
|
||||
, del_remove_cache/2
|
||||
, sync_finish/1
|
||||
, sync_finish/2
|
||||
]).
|
||||
|
||||
-define(MS_TKV_PROC_REMOVE_CACHE(OptTable), {remove_cache, OptTable}).
|
||||
-define(REMOVE_CACHE_DICT_KEY(Index), {remove_cache, Index}).
|
||||
|
||||
%%%===================================================================
|
||||
%%% API
|
||||
%%%===================================================================
|
||||
%% @doc
|
||||
%% 删除表所有数据
|
||||
%% @end
|
||||
delete_all_objects(#{ tab:=Table,opt_tab:=OptTable,handler:=Handler,safe_mode:=SafeMode }) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:delete_all_objects(OptTable),
|
||||
%% 删除数据库所有数据
|
||||
Handler:delete_all(?UPDATE_POOL),
|
||||
ms_cache_dao_sync_gs:delete_all_objects(Table),
|
||||
?OK.
|
||||
|
||||
%% @doc
|
||||
%% 查询待同步的数据
|
||||
%% @end
|
||||
lookup(#{ opt_tab:=OptTable, safe_mode:=SafeMode }, Key) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:lookup(OptTable,Key).
|
||||
|
||||
%% @doc
|
||||
%% 插入待同步的数据
|
||||
%% @end
|
||||
insert(#{ opt_tab:=OptTable, safe_mode:=SafeMode }, Record) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:insert(OptTable,Record),
|
||||
?OK.
|
||||
|
||||
%% @doc
|
||||
%% 删除待同步的数据
|
||||
%% @end
|
||||
delete(#{ opt_tab:=OptTable,safe_mode:=SafeMode }, Key) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:delete(OptTable,Key),
|
||||
?OK.
|
||||
|
||||
%% @doc
|
||||
%% 关闭/删除待同步的数据表
|
||||
%% @end
|
||||
close(#{ opt_tab:=OptTable,safe_mode:=SafeMode }) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:close(OptTable),
|
||||
?OK.
|
||||
|
||||
%% @doc
|
||||
%% 检测待同步数据量是否到设定阈值，是则同步，不是则不处理
|
||||
%% @end
|
||||
get_sync_num(#{ opt_tab:=OptTable,safe_mode:=SafeMode }) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
DbCacheMod:table_info(OptTable, size).
|
||||
|
||||
%% @doc
|
||||
%% 检测待同步数据量是否到设定阈值，是则同步，不是则不处理
|
||||
%% @end
|
||||
is_sync_now(#{ opt_tab:=OptTable,safe_mode:=SafeMode,db_save_count:=DbSaveCount }) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
SyncNum = DbCacheMod:table_info(OptTable, size),
|
||||
SyncNum >= DbSaveCount.
|
||||
|
||||
%% @doc
|
||||
%% 指定单笔数据同步到数据库
|
||||
%% @end
|
||||
try_sync_one_to_db(#{ tab:=Table,opt_tab:=OptTable,safe_mode:=SafeMode }, OptKey) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
case DbCacheMod:lookup(OptTable, OptKey) of
|
||||
[] ->
|
||||
ok;
|
||||
[OptRecord] ->
|
||||
ms_cache_dao_sync_gs:sync_one_to_db(Table, OptRecord)
|
||||
%% todo，这里不删除记录，确保数据不丢失；注意这种情况数据库插入要兼容存在时更新（目前是支持的）
|
||||
%% DbCacheMod:delete(OptTable, OptKey)
|
||||
end,
|
||||
ok.
|
||||
|
||||
%% @doc
|
||||
%% 指定单笔数据同步到数据库 todo 暂时没有这个方法的应用场景，先改成多次单笔数据同步到数据库
|
||||
%% @end
|
||||
try_sync_list_to_db(State, OptKeys) ->
|
||||
lists:foreach(fun(OptKey) -> try_sync_one_to_db(State, OptKey) end, OptKeys),
|
||||
ok.
|
||||
|
||||
|
||||
%% @doc
|
||||
%% 所有待同步数据同步到数据库
|
||||
%% @end
|
||||
do_sync(#{ tab:=Table,opt_tab:=OptTable }, SyncType) ->
|
||||
ms_cache_dao_sync_gs:sync(Table, OptTable, SyncType),
|
||||
ok.
|
||||
|
||||
%% @doc
|
||||
%% 获取侦听数据同步事件的进程
|
||||
%% @end
|
||||
sync_finish(OptTable) ->
|
||||
ms_cache_tkv:foreach(
|
||||
?MS_TKV_PROC_REMOVE_CACHE(OptTable),
|
||||
fun(Index, Pid) ->
|
||||
?ERASE_FROM_PROC_DICT(?REMOVE_CACHE_DICT_KEY(Index)),
|
||||
sync_finish(Pid, Index)
|
||||
end
|
||||
),
|
||||
ms_cache_tkv:delete(?MS_TKV_PROC_REMOVE_CACHE(OptTable)),
|
||||
?OK.
|
||||
|
||||
sync_finish(Pid, Index) ->
|
||||
erlang:send(Pid, {sync_finish, Index}, [noconnect]),
|
||||
?OK.
|
||||
|
||||
%%%===================================================================
|
||||
%%% Internal functions
|
||||
%%%===================================================================
|
||||
set_remove_cache(OptTable, Index, Pid) ->
|
||||
?PUT_TO_PROC_DICT(?REMOVE_CACHE_DICT_KEY(Index), OptTable),
|
||||
ms_cache_tkv:add(?MS_TKV_PROC_REMOVE_CACHE(OptTable), Index, Pid),
|
||||
?OK.
|
||||
|
||||
del_remove_cache(_OptTable, Index) ->
|
||||
case ?ERASE_FROM_PROC_DICT(?REMOVE_CACHE_DICT_KEY(Index)) of
|
||||
?UNDEFINED ->
|
||||
?SKIP;
|
||||
OldOptTable ->
|
||||
ms_cache_tkv:delete(?MS_TKV_PROC_REMOVE_CACHE(OldOptTable), Index)
|
||||
end,
|
||||
?OK.
|
||||
@@ -0,0 +1,32 @@
|
||||
-module(ms_cache_dao_merge_gs_sup).
|
||||
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% API
|
||||
-export([start_link/0]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
%% Helper macro for declaring children of supervisor
|
||||
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, infinity, Type, [I]}).
|
||||
|
||||
%% ===================================================================
|
||||
%% API functions
|
||||
%% ===================================================================
|
||||
|
||||
start_link() ->
|
||||
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
|
||||
|
||||
%% ===================================================================
|
||||
%% Supervisor callbacks
|
||||
%% ===================================================================
|
||||
|
||||
init([]) ->
|
||||
{
|
||||
ok,
|
||||
{
|
||||
{simple_one_for_one, 5, 10},
|
||||
[?CHILD(ms_cache_dao_merge_gs, worker)]
|
||||
}
|
||||
}.
|
||||
@@ -0,0 +1,212 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2019, <COMPANY>
|
||||
%%% @doc
|
||||
%%% ms_cache与数据库交互的进程
|
||||
%%% @end
|
||||
%%% Created : 2019-09-01 00:00:00
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_dao_sync_gs).
|
||||
-include("base.hrl").
|
||||
-behaviour(gen_server).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% include
|
||||
%%--------------------------------------------------------------------
|
||||
-include("ms_cache.hrl").
|
||||
%% Auto recovered record definitions extracted from decompiled Erlang source.
|
||||
%% Keep record definitions in headers; do not define them directly in .erl files.
|
||||
|
||||
-record(state, {}).
|
||||
%%--------------------------------------------------------------------
|
||||
%% API exports
|
||||
%%--------------------------------------------------------------------
|
||||
|
||||
-export([
|
||||
start_link/4
|
||||
, set_batch_len/2 %% 设置批量同步数据sql语句的最大长度
|
||||
, sync/3
|
||||
, sync_one_to_db/2
|
||||
, delete_all_objects/1
|
||||
]).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% gen_server callbacks
|
||||
%%--------------------------------------------------------------------
|
||||
-export([
|
||||
init/1
|
||||
,handle_call/3
|
||||
,handle_cast/2
|
||||
,handle_info/2
|
||||
,terminate/2
|
||||
,code_change/3
|
||||
]).
|
||||
|
||||
%% 缓存数据表与数据库交互的进程
|
||||
-define(SERVER(Table), list_to_atom(atom_to_list(?MODULE) ++ "_" ++ atom_to_list(Table))).
|
||||
|
||||
|
||||
%%%===================================================================
|
||||
%%% API
|
||||
%%%===================================================================
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% Starts the server
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(start_link(Table :: atom(), BatchLen :: integer(), RangeMemSize:: integer(), SafeMode :: boolean()) ->
|
||||
{ok, Pid :: pid()} | ignore | {error, Reason :: term()}).
|
||||
start_link(Table, BatchLen, RangeMemSize, SafeMode) ->
|
||||
gen_server:start_link({local, ?SERVER(Table)}, ?MODULE, [Table, BatchLen, RangeMemSize, SafeMode], []).
|
||||
|
||||
%%%===================================================================
|
||||
%%% gen_server callbacks
|
||||
%%%===================================================================
|
||||
set_batch_len(Table, BatchLen) ->
|
||||
gen_server:cast(?SERVER(Table), {set_batch_len, BatchLen}).
|
||||
|
||||
%% 保存到数据库
|
||||
sync(Table, OptTable, SyncType) ->
|
||||
gen_server:cast(?SERVER(Table), {sync, OptTable, SyncType}).
|
||||
|
||||
%% 保存单个数据到数据库
|
||||
sync_one_to_db(Table, OptRecord) ->
|
||||
gen_server:cast(?SERVER(Table), {sync_one_to_db, OptRecord}).
|
||||
|
||||
delete_all_objects(Table) ->
|
||||
gen_server:cast(?SERVER(Table), {delete_all_objects}).
|
||||
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Initializes the server
|
||||
%%
|
||||
%% @spec init(Args) -> {ok, State} |
|
||||
%% {ok, State, Timeout} |
|
||||
%% ignore |
|
||||
%% {stop, Reason}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(init(Args :: term()) ->
|
||||
{ok, State :: #state{}} | {ok, State :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term()} | ignore).
|
||||
|
||||
init([Table, BatchLen, RangeMemSize, SafeMode]) ->
|
||||
?DEBUG(?MODULE, "~p init", [?SERVER(Table)]),
|
||||
erlang:process_flag(trap_exit, true),
|
||||
State = #{
|
||||
tab => Table,
|
||||
handler => ?TABLE_HANDLER(Table),
|
||||
batch_len => BatchLen,
|
||||
range_mem_size => RangeMemSize,
|
||||
safe_mode => SafeMode
|
||||
},
|
||||
ms_cache_dao_sync_gs_lib:create_table(Table, SafeMode),
|
||||
{ok, State}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling call messages
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_call(Request :: term(), From :: {pid(), Tag :: term()},
|
||||
State :: #state{}) ->
|
||||
{reply, Reply :: term(), NewState :: #state{}} |
|
||||
{reply, Reply :: term(), NewState :: #state{}, timeout() | hibernate} |
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), Reply :: term(), NewState :: #state{}} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_call(Request, From, State) ->
|
||||
?TRY_CALL(Request, From, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling cast messages
|
||||
%%
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_cast(Request :: term(), State :: #state{}) ->
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_cast(Request, State) ->
|
||||
?TRY_CAST(Request, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Handling all non call/cast messages
|
||||
%%
|
||||
%% @spec handle_info(Info, State) -> {noreply, State} |
|
||||
%% {noreply, State, Timeout} |
|
||||
%% {stop, Reason, State}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(handle_info(Info :: timeout() | term(), State :: #state{}) ->
|
||||
{noreply, NewState :: #state{}} |
|
||||
{noreply, NewState :: #state{}, timeout() | hibernate} |
|
||||
{stop, Reason :: term(), NewState :: #state{}}).
|
||||
handle_info(Info, State) ->
|
||||
?TRY_INFO(Info, State).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% This function is called by a gen_server when it is about to
|
||||
%% terminate. It should be the opposite of Module:init/1 and do any
|
||||
%% necessary cleaning up. When it returns, the gen_server terminates
|
||||
%% with Reason. The return value is ignored.
|
||||
%%
|
||||
%% @spec terminate(Reason, State) -> void()
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(terminate(Reason :: (normal | shutdown | {shutdown, term()} | term()),
|
||||
State :: #state{}) -> term()).
|
||||
terminate(Reason, #{ tab:=Table } = _State) ->
|
||||
?DEBUG(?MODULE, "~p terminate, Reason = ~p, Table = ~p", [?SERVER(Table), Reason, Table]),
|
||||
ok.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @private
|
||||
%% @doc
|
||||
%% Convert process state when code is changed
|
||||
%%
|
||||
%% @spec code_change(OldVsn, State, Extra) -> {ok, NewState}
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec(code_change(OldVsn :: term() | {down, term()}, State :: #state{},
|
||||
Extra :: term()) ->
|
||||
{ok, NewState :: #state{}} | {error, Reason :: term()}).
|
||||
code_change(_OldVsn, State, _Extra) ->
|
||||
{ok, State}.
|
||||
|
||||
%%%===================================================================
|
||||
%%% gen_server callbacks Internal
|
||||
%%%===================================================================
|
||||
do_call(_Request, _From, State) ->
|
||||
{reply, ok, State}.
|
||||
|
||||
do_cast({set_batch_len, BatchLen}, State) ->
|
||||
{noreply, State#{batch_len:=BatchLen}};
|
||||
do_cast({sync, OptTable, SyncType}, State) ->
|
||||
ms_cache_dao_sync_gs_lib:do_sync(State, OptTable, SyncType),
|
||||
{noreply, State};
|
||||
do_cast({sync_one_to_db, OptRecord}, State) ->
|
||||
ms_cache_dao_sync_gs_lib:try_sync_one_to_db(State, OptRecord),
|
||||
{noreply, State};
|
||||
do_cast({delete_all_objects}, State) ->
|
||||
ms_cache_dao_sync_gs_lib:delete_all_objects(State),
|
||||
{noreply, State}.
|
||||
|
||||
%% 手动同步同步单一数据到数据库
|
||||
do_info(_Info, State) ->
|
||||
{noreply, State}.
|
||||
%%%===================================================================
|
||||
%%% Internal functions
|
||||
%%%===================================================================
|
||||
@@ -0,0 +1,314 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2023, <COMPANY>
|
||||
%%% @doc
|
||||
%%%
|
||||
%%% @end
|
||||
%%% Created : 03. 7月 2023 20:18
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_dao_sync_gs_lib).
|
||||
-include("base.hrl").
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% API
|
||||
-export([
|
||||
do_sync/3
|
||||
, try_sync_one_to_db/2
|
||||
, delete_all_objects/1
|
||||
, create_table/2
|
||||
]).
|
||||
|
||||
%%%===================================================================
|
||||
%%% API
|
||||
%%%===================================================================
|
||||
|
||||
%% @doc
|
||||
%% 所有待同步数据同步到数据库
|
||||
%% @end
|
||||
do_sync(#{ tab:=Table,handler:=Handler,batch_len:=BatchLen, range_mem_size:=RangeMemberSize, safe_mode:=SafeMode }, OptTable, SyncType) ->
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
SyncNum = DbCacheMod:table_info(OptTable, size),
|
||||
case Handler:sync_mode() of
|
||||
batch ->
|
||||
batch_sync_to_db(Table, OptTable, Handler, BatchLen, RangeMemberSize, SyncType, DbCacheMod);
|
||||
_ ->
|
||||
one_by_one_sync_to_db(Table, OptTable, Handler, SyncType, DbCacheMod)
|
||||
end,
|
||||
|
||||
case DbCacheMod:table_info(OptTable, size) of
|
||||
LeftNum when LeftNum > 0 ->
|
||||
filelib:ensure_dir("log/cache/"),
|
||||
FileName = "log/cache/" ++ erlang:atom_to_list(OptTable) ++ "-" ++ erlang:integer_to_list(ms_cache_util:now_ms()),
|
||||
?ERROR(?MODULE, "sync left, OptTable = ~p, SyncNum = ~p, LeftNum = ~p, FileName = ~p", [OptTable, SyncNum, LeftNum, FileName]),
|
||||
ets:tab2file(OptTable, FileName);
|
||||
_ ->
|
||||
?SKIP
|
||||
end,
|
||||
|
||||
case SyncType /= terminate of
|
||||
?TRUE ->
|
||||
do_sync_finish_notify(Table, OptTable);
|
||||
?FALSE ->
|
||||
?INFO(?MODULE, "terminate, Table = ~p, SyncNum = ~p", [Table, SyncNum])
|
||||
end,
|
||||
?OK.
|
||||
|
||||
%% @doc
|
||||
%% 指定单笔数据同步到数据库
|
||||
%% @end
|
||||
try_sync_one_to_db(#{ tab:=Table,handler:=Handler }, #ms_cache_opt{opt='insert',opt_rec=OptRecord,key=OptKey}) ->
|
||||
try
|
||||
%% Handler:insert(?UPDATE_POOL, OptRecord)
|
||||
case OptRecord of
|
||||
?NONE ->
|
||||
TableData = ms_cache_util:get_table_name_by_key(Table, data, OptKey),
|
||||
case ms_cache_edb:lookup(TableData, OptKey) of
|
||||
[] ->
|
||||
?OK;
|
||||
[OptRecord1] ->
|
||||
Handler:insert(?UPDATE_POOL, OptRecord1)
|
||||
end;
|
||||
_ ->
|
||||
Handler:insert(?UPDATE_POOL, OptRecord)
|
||||
end
|
||||
catch _ErrCode:ErrMsg:Trace ->
|
||||
%% todo 失败了怎么办，重新返还给ms_cache_dao_merge_gs? 好像也不行，比如本次是update，但是ms_cache_dao_merge_gs已经收到了delete消息，这样会有问题
|
||||
?ERROR(?MODULE, "insert error, Table = ~p, OptRecord = ~p, ErrMsg = ~p, Trace = ~p",
|
||||
[Table, OptRecord, ErrMsg, Trace])
|
||||
end,
|
||||
?OK;
|
||||
try_sync_one_to_db(#{ tab:=Table,handler:=Handler }, #ms_cache_opt{opt='update',opt_rec=OptRecord,key=OptKey}) ->
|
||||
try
|
||||
%% Handler:update(?UPDATE_POOL, OptRecord)
|
||||
case OptRecord of
|
||||
?NONE ->
|
||||
TableData = ms_cache_util:get_table_name_by_key(Table, data, OptKey),
|
||||
case ms_cache_edb:lookup(TableData, OptKey) of
|
||||
[] ->
|
||||
?OK;
|
||||
[OptRecord1] ->
|
||||
Handler:update(?UPDATE_POOL, OptRecord1)
|
||||
end;
|
||||
_ ->
|
||||
Handler:update(?UPDATE_POOL, OptRecord)
|
||||
end
|
||||
catch _ErrCode:ErrMsg:Trace ->
|
||||
%% todo 失败了怎么办，重新返还给ms_cache_dao_merge_gs? 好像也不行，比如本次是update，但是ms_cache_dao_merge_gs已经收到了delete消息，这样会有问题
|
||||
?ERROR(?MODULE, "update error, Table = ~p, OptRecord = ~p, ErrMsg = ~p, Trace = ~p",
|
||||
[Table, OptRecord, ErrMsg, Trace])
|
||||
end,
|
||||
?OK;
|
||||
try_sync_one_to_db(#{ tab:=Table,handler:=Handler }, #ms_cache_opt{opt='delete',key=OptKey}) ->
|
||||
try
|
||||
Handler:delete(?UPDATE_POOL, OptKey)
|
||||
catch _ErrCode:ErrMsg:Trace ->
|
||||
%% todo 失败了怎么办，重新返还给ms_cache_dao_merge_gs? 好像也不行，比如本次是update，但是ms_cache_dao_merge_gs已经收到了delete消息，这样会有问题
|
||||
?ERROR(?MODULE, "delete error, Table = ~p, OptKey = ~p, ErrMsg = ~p, Trace = ~p",
|
||||
[Table, OptKey, ErrMsg, Trace])
|
||||
end,
|
||||
?OK.
|
||||
|
||||
delete_all_objects(#{ handler := Handler }) ->
|
||||
%% 删除数据库所有数据
|
||||
Handler:delete_all(?UPDATE_POOL),
|
||||
?OK.
|
||||
|
||||
%%%===================================================================
|
||||
%%% Internal functions
|
||||
%%%===================================================================
|
||||
sync_list_to_db([Record | T], Opt, Table, OptTable, Handler, CacheDbMod) ->
|
||||
try
|
||||
Key =
|
||||
case Opt of
|
||||
'delete' ->
|
||||
Record;
|
||||
_ ->
|
||||
erlang:element(2, Record)
|
||||
end,
|
||||
sync_one_to_db(#ms_cache_opt{key=Key,opt=Opt,opt_rec=Record}, OptTable, Handler, CacheDbMod)
|
||||
catch _ErrCode:ErrMsg:Trace ->
|
||||
?ERROR(?MODULE, "Table = ~p, Record = ~p, ErrMsg = ~p, OptTable : ~p, Trace = ~p",
|
||||
[Table, Record, ErrMsg, OptTable, Trace])
|
||||
end,
|
||||
sync_list_to_db(T, Opt, Table, OptTable, Handler, CacheDbMod);
|
||||
sync_list_to_db([], _Opt, _Table, _OptTable, _Handler, _CacheDbMod) ->
|
||||
ok.
|
||||
|
||||
sync_one_to_db(#ms_cache_opt{key=Key, opt='insert',opt_rec=Record}, OptTable, Handler, CacheDbMod) ->
|
||||
Handler:insert(?UPDATE_POOL, Record),
|
||||
CacheDbMod:delete(OptTable, Key),
|
||||
ok;
|
||||
sync_one_to_db(#ms_cache_opt{key=Key, opt='update',opt_rec=Record}, OptTable, Handler, CacheDbMod) ->
|
||||
Handler:update(?UPDATE_POOL, Record),
|
||||
CacheDbMod:delete(OptTable, Key),
|
||||
ok;
|
||||
sync_one_to_db(#ms_cache_opt{key=Key, opt='delete'}, OptTable, Handler, CacheDbMod) ->
|
||||
Handler:delete(?UPDATE_POOL, Key),
|
||||
CacheDbMod:delete(OptTable, Key),
|
||||
ok.
|
||||
|
||||
%% 逐条同步到数据库
|
||||
one_by_one_sync_to_db(Table, OptTable, Handler, Type, CacheDbMod) ->
|
||||
SyncNum = CacheDbMod:table_info(OptTable, size),
|
||||
case SyncNum > 0 of
|
||||
true ->
|
||||
Fun =
|
||||
fun(OptRecord) ->
|
||||
try
|
||||
sync_one_to_db(OptRecord, OptTable, Handler, CacheDbMod)
|
||||
catch _ErrCode:ErrMsg:Trace ->
|
||||
?ERROR(?MODULE, "Table = ~p, OptRecord = ~p, ErrMsg = ~p, OptTable : ~p, Trace = ~p",
|
||||
[Table, OptRecord, ErrMsg, OptTable, Trace])
|
||||
end
|
||||
end,
|
||||
BeginSyncTime = ms_cache_util:now_ms(),
|
||||
|
||||
CacheDbMod:foreach(OptTable, Fun),
|
||||
|
||||
EndSyncTime = ms_cache_util:now_ms(),
|
||||
case EndSyncTime-BeginSyncTime > 5000 of
|
||||
true ->
|
||||
?INFO(?MODULE, "db_sync_time_info, Type = ~p, Table = ~p, SyncNum = ~p, SyncTime = ~p",
|
||||
[Type, Table, SyncNum, EndSyncTime-BeginSyncTime]);
|
||||
false ->
|
||||
ok
|
||||
end,
|
||||
ok;
|
||||
false ->
|
||||
ok
|
||||
end.
|
||||
|
||||
%% 批量同步到数据库
|
||||
batch_sync_to_db(Table, OptTable, Handler, BatchLen, RangeMemberSize, Type, CacheDbMod) ->
|
||||
SyncNum = CacheDbMod:table_info(OptTable, size),
|
||||
case SyncNum > 0 of
|
||||
true ->
|
||||
BeginSyncTime = ms_cache_util:now_ms(),
|
||||
{ok, InsertAcc, UpdateAcc, DeleteAcc} = get_sync_group_datas(Table, OptTable, CacheDbMod),
|
||||
batch_insert(Table, OptTable, InsertAcc, Handler, BatchLen, CacheDbMod),
|
||||
batch_update(Table, OptTable, UpdateAcc, Handler, BatchLen, CacheDbMod),
|
||||
batch_delete(Table, OptTable, DeleteAcc, Handler, RangeMemberSize, CacheDbMod),
|
||||
|
||||
EndSyncTime = ms_cache_util:now_ms(),
|
||||
case EndSyncTime-BeginSyncTime > 5000 of
|
||||
true ->
|
||||
?INFO(?MODULE, "db_sync_time_info, Type = ~p, Table = ~p, OptTable = ~p, SyncNum = ~p, SyncTime = ~p",
|
||||
[Type, Table, OptTable, SyncNum, EndSyncTime-BeginSyncTime]);
|
||||
false ->
|
||||
ok
|
||||
end;
|
||||
false ->
|
||||
ok
|
||||
end,
|
||||
ok.
|
||||
|
||||
batch_insert(Table, OptTable, Records, Handler, BatchLen, CacheDbMod) ->
|
||||
try
|
||||
ok = Handler:batch_insert(?UPDATE_POOL, Records, BatchLen),
|
||||
[CacheDbMod:delete(OptTable, erlang:element(2, Record)) || Record <- Records],
|
||||
ok
|
||||
catch
|
||||
error:{exit, {timeout, _}, _} = ErrMsg -> %% 超时不触发逐笔插入，防止一直超时
|
||||
?ERROR(?MODULE, "batch_insert timeout, Table = ~p, ErrMsg = ~p", [Table, ErrMsg]),
|
||||
ok;
|
||||
_Error:ErrMsg:Trace -> %% 数据库超时可能触发两次操作，理论上也没什么问题
|
||||
?ERROR(?MODULE, "batch_insert error, try sync_list_to_db, Table = ~p, ErrMsg = ~p, stacktrace = ~p",
|
||||
[Table, ErrMsg, Trace]),
|
||||
sync_list_to_db(Records, insert, Table, OptTable, Handler, CacheDbMod),
|
||||
ok
|
||||
end.
|
||||
|
||||
batch_update(Table, OptTable, Records, Handler, BatchLen, CacheDbMod) ->
|
||||
try
|
||||
%% 使用batch_insert來替代batch_update
|
||||
ok = Handler:batch_insert(?UPDATE_POOL, Records, BatchLen),
|
||||
[CacheDbMod:delete(OptTable, erlang:element(2, Record)) || Record <- Records],
|
||||
ok
|
||||
catch
|
||||
error:{exit, {timeout, _}, _} = ErrMsg -> %% 超时不触发逐笔插入，防止一直超时
|
||||
?ERROR(?MODULE, "batch_update timeout, Table = ~p, ErrMsg = ~p", [Table, ErrMsg]),
|
||||
ok;
|
||||
_Error:ErrMsg:Trace -> %% 数据库超时可能触发两次操作，理论上也没什么问题
|
||||
?ERROR(?MODULE, "batch_update error, try sync_list_to_db, Table = ~p, ErrMsg = ~p, stacktrace = ~p",
|
||||
[Table, ErrMsg, Trace]),
|
||||
sync_list_to_db(Records, update, Table, OptTable, Handler, CacheDbMod),
|
||||
ok
|
||||
end.
|
||||
|
||||
batch_delete(Table, OptTable, Keys, Handler, BatchLen, CacheDbMod) ->
|
||||
try
|
||||
ok = Handler:batch_delete(?UPDATE_POOL, Keys, BatchLen),
|
||||
[CacheDbMod:delete(OptTable, Key) || Key <- Keys],
|
||||
ok
|
||||
catch
|
||||
error:{exit, {timeout, _}, _} = ErrMsg -> %% 超时不触发逐笔插入，防止一直超时
|
||||
?ERROR(?MODULE, "batch_delete timeout, Table = ~p, ErrMsg = ~p", [Table, ErrMsg]),
|
||||
ok;
|
||||
_Error:ErrMsg:Trace -> %% 数据库超时可能触发两次操作，理论上也没什么问题
|
||||
?ERROR(?MODULE, "batch_delete error, try sync_list_to_db, Table = ~p, ErrMsg = ~p, stacktrace = ~p",
|
||||
[Table, ErrMsg, Trace]),
|
||||
sync_list_to_db(Keys, delete, Table, OptTable, Handler, CacheDbMod),
|
||||
ok
|
||||
end.
|
||||
|
||||
%% 分组所有插入 更新 删除的记录
|
||||
get_sync_group_datas(Table, OptTable, CacheDbMod) ->
|
||||
Fun =
|
||||
fun(E, {InsertAcc, UpdateAcc, DeleteAcc}) ->
|
||||
case E of
|
||||
#ms_cache_opt{ key=Key,opt='insert',opt_rec = ?NONE } ->
|
||||
DataTable = ms_cache_util:get_table_name_by_key(Table, data, Key),
|
||||
case ms_cache_edb:lookup(DataTable, Key) of
|
||||
[] -> %% 已经被删除，后续会收到delete操作
|
||||
{InsertAcc, UpdateAcc, DeleteAcc};
|
||||
[R] ->
|
||||
{[R | InsertAcc], UpdateAcc, DeleteAcc}
|
||||
end;
|
||||
#ms_cache_opt{ key=Key,opt='update',opt_rec = ?NONE } ->
|
||||
DataTable = ms_cache_util:get_table_name_by_key(Table, data, Key),
|
||||
case ms_cache_edb:lookup(DataTable, Key) of
|
||||
[] -> %% 已经被删除，后续会收到delete操作
|
||||
{InsertAcc, UpdateAcc, DeleteAcc};
|
||||
[R] ->
|
||||
{InsertAcc, [R | UpdateAcc], DeleteAcc}
|
||||
end;
|
||||
#ms_cache_opt{ opt='insert',opt_rec=R } ->
|
||||
{[R | InsertAcc], UpdateAcc, DeleteAcc};
|
||||
#ms_cache_opt{ opt='update',opt_rec=R } ->
|
||||
{InsertAcc, [R | UpdateAcc], DeleteAcc};
|
||||
#ms_cache_opt{ opt='delete',key=Key } ->
|
||||
{InsertAcc, UpdateAcc, [Key | DeleteAcc]}
|
||||
end
|
||||
end,
|
||||
{InsertAcc1, UpdateAcc1, DeleteAcc1} = CacheDbMod:foldl(OptTable, Fun, {[], [] ,[]}),
|
||||
{ok, InsertAcc1, UpdateAcc1, DeleteAcc1}.
|
||||
|
||||
%% 通知侦听finish_notify的进程（即缓存进程ms_cache_gs）同步完成
|
||||
do_sync_finish_notify(Table, OptTable) ->
|
||||
ms_cache_dao_merge_gs:sync_finish(Table, OptTable),
|
||||
ok.
|
||||
|
||||
%% 创建待同步数据表
|
||||
%% @end
|
||||
create_table(Table, SafeMode) ->
|
||||
%% 同步表使用dets存储，不用mnesia的原因如下
|
||||
%% 其他表（数据表、加载表和过期表使用mnesia临时表）
|
||||
%% 启动时调用了mnesia:change_table_copy_type(schema, node(), ram_copies)，之后创建的mnesia表都是内存临时表
|
||||
%% 而同步表，在数据库连不上 /节点挂掉的情况，需要存储，减少数据的丢失，所以不能用mnesia临时表
|
||||
%% 如果mnesia不指定为临时表，则其他表的表结构会存在mnesia；如果结构变动要删除旧表再创新表，启动服务器非常慢
|
||||
%% todo 也有可能是对mnesia表类型的操作还不够深入，使用不正确导致，后面看看是否有更合适的方法，暂时先用dets实现
|
||||
%% 这里使用ets是因为ets的性能比mnesia高出一个数量级（5被差距左右）
|
||||
DbCacheMod = ms_cache_util:get_cache_opt_mod(SafeMode),
|
||||
lists:foreach(
|
||||
fun(OT) ->
|
||||
DbCacheMod:create_table(OT, []),
|
||||
Size = DbCacheMod:table_info(OT, size),
|
||||
case Size > 0 of
|
||||
?TRUE -> %% 修复数据(只有SafeMode=true，dets表的模式下才会同步)
|
||||
?INFO(?MODULE, "repair datas, Table = ~p, size = ~p", [Table, Size]),
|
||||
ms_cache_dao_sync_gs:sync(Table, OT, repair_sync);
|
||||
?FALSE ->
|
||||
?SKIP
|
||||
end
|
||||
end, ?OPT_TABLE_LIST(Table)
|
||||
),
|
||||
?OK.
|
||||
@@ -0,0 +1,32 @@
|
||||
-module(ms_cache_dao_sync_gs_sup).
|
||||
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% API
|
||||
-export([start_link/0]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
%% Helper macro for declaring children of supervisor
|
||||
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, infinity, Type, [I]}).
|
||||
|
||||
%% ===================================================================
|
||||
%% API functions
|
||||
%% ===================================================================
|
||||
|
||||
start_link() ->
|
||||
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
|
||||
|
||||
%% ===================================================================
|
||||
%% Supervisor callbacks
|
||||
%% ===================================================================
|
||||
|
||||
init([]) ->
|
||||
{
|
||||
ok,
|
||||
{
|
||||
{simple_one_for_one, 5, 10},
|
||||
[?CHILD(ms_cache_dao_sync_gs, worker)]
|
||||
}
|
||||
}.
|
||||
@@ -0,0 +1,190 @@
|
||||
%%% -------------------------------------------------------------------
|
||||
%%% Author : wangyalun
|
||||
%%% Description : dets缓存数据库
|
||||
%%% Created : 2019-01-14
|
||||
%%% -------------------------------------------------------------------
|
||||
-module(ms_cache_ddb).
|
||||
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-include_lib("stdlib/include/ms_transform.hrl").
|
||||
|
||||
-define(TAB_DEFAULT_OPTS, [{ram_copies, [node()]}]).
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
-export([
|
||||
init/0
|
||||
, lookup/2
|
||||
, index_read_keys/3
|
||||
, index_read/3
|
||||
, index_match_object/3
|
||||
, lookup_element/3
|
||||
, update_element/3
|
||||
, insert/2
|
||||
, insert_new/2
|
||||
, delete/2
|
||||
, create_table/2
|
||||
, delete_all_objects/1
|
||||
, table_info/2
|
||||
, foreach/2
|
||||
, foreach/3
|
||||
, foldl/3
|
||||
, all_keys/1
|
||||
, tab2list/1
|
||||
, add_table_copy/3
|
||||
, wait_for_tables/2
|
||||
, member/2
|
||||
, close/1
|
||||
]).
|
||||
|
||||
init() ->
|
||||
ok.
|
||||
|
||||
lookup(Tab, Key) ->
|
||||
dets:lookup(Tab, Key).
|
||||
|
||||
index_read_keys(Table, Key, Pos) ->
|
||||
FoldlFun =
|
||||
fun(Val, Acc) ->
|
||||
case erlang:element(Pos, Val) of
|
||||
Key ->
|
||||
[erlang:element(2, Val) | Acc];
|
||||
_ ->
|
||||
Acc
|
||||
end
|
||||
end,
|
||||
foldl(Table, FoldlFun, []).
|
||||
|
||||
index_read(Table, Key, Pos) ->
|
||||
FoldlFun =
|
||||
fun(Val, Acc) ->
|
||||
case erlang:element(Pos, Val) of
|
||||
Key ->
|
||||
[Val | Acc];
|
||||
_ ->
|
||||
Acc
|
||||
end
|
||||
end,
|
||||
foldl(Table, FoldlFun, []).
|
||||
|
||||
index_match_object(Table, Pattern, _Pos) ->
|
||||
FoldlFun =
|
||||
fun(Rec, Acc) ->
|
||||
case lists:all(fun({P, K}) -> erlang:element(P, Rec) == K end, Pattern) of
|
||||
true ->
|
||||
[Rec | Acc];
|
||||
false ->
|
||||
Acc
|
||||
end
|
||||
end,
|
||||
foldl(Table, FoldlFun, []).
|
||||
|
||||
lookup_element(Table, Key, Pos) ->
|
||||
[Record] = dets:lookup(Table, Key),
|
||||
erlang:element(Pos, Record).
|
||||
|
||||
update_element(Table, Key, {Pos, Value}) ->
|
||||
case dets:lookup(Table, Key) of
|
||||
[Record] ->
|
||||
NewRecord = erlang:setelement(Pos, Record, Value),
|
||||
dets:insert(Table, NewRecord),
|
||||
true;
|
||||
_ ->
|
||||
false
|
||||
end;
|
||||
update_element(Table, Key, PosValueList) ->
|
||||
case dets:lookup(Table, Key) of
|
||||
[Record] ->
|
||||
NewRecord = lists:foldl(fun(Pos, Value, Acc) -> erlang:setelement(Pos, Acc, Value) end, Record, PosValueList),
|
||||
dets:insert(Table, NewRecord),
|
||||
true;
|
||||
_ ->
|
||||
false
|
||||
end.
|
||||
|
||||
insert(Tab, Record) ->
|
||||
dets:insert(Tab, Record).
|
||||
|
||||
insert_new(Tab, Record) ->
|
||||
dets:insert_new(Tab, Record).
|
||||
|
||||
delete(Tab, Key) ->
|
||||
dets:delete(Tab, Key).
|
||||
|
||||
member(Tab, Key) ->
|
||||
dets:member(Tab, Key).
|
||||
|
||||
create_table(Tab, _Opts) ->
|
||||
FileName = "dets/" ++ erlang:atom_to_list(Tab),
|
||||
dets:open_file(Tab, [{type, set}, {keypos, 2}, {file, FileName}]).
|
||||
|
||||
delete_all_objects(Tab) ->
|
||||
dets:delete_all_objects(Tab).
|
||||
|
||||
table_info(Tab, Type) ->
|
||||
dets:info(Tab, Type).
|
||||
|
||||
foreach(Tab, Fun) when is_function(Fun, 1) ->
|
||||
dets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, dets:first(Tab))
|
||||
after
|
||||
dets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table') ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key) ->
|
||||
[Element] = dets:lookup(Tab, Key),
|
||||
Fun(Element),
|
||||
foreach_1(Tab, Fun, dets:next(Tab, Key)).
|
||||
|
||||
foreach(Tab, Fun, Limit) when is_function(Fun, 1) ->
|
||||
dets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, dets:first(Tab), Limit)
|
||||
after
|
||||
dets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table', _N) ->
|
||||
ok;
|
||||
foreach_1(_Tab, _Fun, _, N) when N =< 0 ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key, N) ->
|
||||
[Element] = dets:lookup(Tab, Key),
|
||||
Fun(Element),
|
||||
foreach_1(Tab, Fun, dets:next(Tab, Key), N-1).
|
||||
|
||||
foldl(Tab, Fun, Acc) when is_function(Fun, 2) ->
|
||||
dets:foldl(Fun, Acc, Tab).
|
||||
|
||||
all_keys(Tab) ->
|
||||
MS = ets:fun2ms(fun(Rec) -> element(2, Rec) end),
|
||||
dets:select(Tab, MS).
|
||||
|
||||
tab2list(Tab) ->
|
||||
MS = ets:fun2ms(fun(Rec) -> Rec end),
|
||||
dets:select(Tab, MS).
|
||||
|
||||
add_table_copy(_Tab, _CacheNode, _TabType) ->
|
||||
ok.
|
||||
|
||||
wait_for_tables(_Tables, _TimeOut) ->
|
||||
ok.
|
||||
|
||||
close(Table) ->
|
||||
SyncNum = table_info(Table, size),
|
||||
dets:close(Table),
|
||||
case SyncNum > 0 of
|
||||
true ->
|
||||
ok;
|
||||
false ->
|
||||
FileName = "dets/" ++ erlang:atom_to_list(Table),
|
||||
file:delete(FileName)
|
||||
end.
|
||||
@@ -0,0 +1,308 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2023, <COMPANY>
|
||||
%%% @doc
|
||||
%%%
|
||||
%%% @end
|
||||
%%% Created : 27. 11月 2023 19:41
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_edb).
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-include_lib("stdlib/include/ms_transform.hrl").
|
||||
|
||||
-define(TAB_DEFAULT_OPTS, [{ram_copies, [node()]}]).
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
-export([
|
||||
init/0
|
||||
, lookup/2
|
||||
, index_read_keys/3
|
||||
, index_read/3
|
||||
, index_match_object/5
|
||||
, lookup_element/3
|
||||
, update_element/3
|
||||
, insert/2
|
||||
, insert_new/2
|
||||
, delete/2
|
||||
, create_table/2
|
||||
, delete_all_objects/1
|
||||
, table_info/2
|
||||
, foreach/2
|
||||
, foreach/3
|
||||
, foldl/3
|
||||
, all_keys/1
|
||||
, tab2list/1
|
||||
, add_table_copy/3
|
||||
, wait_for_tables/2
|
||||
, member/2
|
||||
, close/1
|
||||
, first/1
|
||||
, next/2
|
||||
, last/1
|
||||
, prev/2
|
||||
, slot/2
|
||||
]).
|
||||
|
||||
init() ->
|
||||
ok.
|
||||
|
||||
lookup(Tab, Key) ->
|
||||
ets:lookup(Tab, Key).
|
||||
|
||||
index_read(Table, Index, Pos) ->
|
||||
RecKeys = index_read_keys(Table, Index, Pos),
|
||||
lists:foldl(
|
||||
fun(RecKey, Acc) ->
|
||||
case ets:lookup(Table, RecKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
[Rec] ->
|
||||
[Rec | Acc]
|
||||
end
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
index_match_object(Table, Index, Pos, Pattern, _Match) ->
|
||||
RecKeys = index_read_keys(Table, Index, Pos),
|
||||
lists:foldl(
|
||||
fun(RecKey, Acc) ->
|
||||
Pattern1 = erlang:setelement(2, Pattern, RecKey),
|
||||
ets:safe_fixtable(Table, true),
|
||||
MatchObjects =
|
||||
try
|
||||
ets:match_object(Table, Pattern1)
|
||||
after
|
||||
ets:safe_fixtable(Table, false)
|
||||
end,
|
||||
MatchObjects ++ Acc
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
lookup_element(Table, Key, Pos) ->
|
||||
ets:lookup_element(Table, Key, Pos).
|
||||
|
||||
update_element(Table, Key, UpdateSpec) ->
|
||||
ets:update_element(Table, Key, UpdateSpec).
|
||||
|
||||
insert(Tab, Record) ->
|
||||
Key = erlang:element(2, Record),
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
ets:insert(Tab, Record),
|
||||
insert_index(Tab, Record);
|
||||
[OldRecord] ->
|
||||
ets:insert(Tab, Record),
|
||||
update_index(Tab, OldRecord, Record)
|
||||
end,
|
||||
true.
|
||||
|
||||
insert_new(Tab, Record) ->
|
||||
case ets:insert_new(Tab, Record) of
|
||||
true ->
|
||||
insert_index(Tab, Record),
|
||||
true;
|
||||
false ->
|
||||
false
|
||||
end.
|
||||
|
||||
delete(Tab, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
skip;
|
||||
[Rec] ->
|
||||
ets:delete(Tab, Key),
|
||||
delete_index(Tab, Rec)
|
||||
end,
|
||||
ok.
|
||||
|
||||
|
||||
member(Tab, Key) ->
|
||||
ets:member(Tab, Key).
|
||||
|
||||
%%create_table(Tab, _Opts) ->
|
||||
%% ets:new(Tab, [public, named_table, set, {keypos, 2}, {read_concurrency, true}]).
|
||||
|
||||
create_table(Tab, Opts) ->
|
||||
TableType = proplists:get_value(type, Opts, set),
|
||||
IndexList1 = proplists:get_value(index, Opts, []),
|
||||
ets:new(Tab, [public, named_table, TableType, {keypos, 2}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
create_index_table(Tab, IndexList1),
|
||||
set_index_list(Tab, IndexList1),
|
||||
ok.
|
||||
|
||||
delete_all_objects(Tab) ->
|
||||
ets:delete_all_objects(Tab),
|
||||
delete_index_all_objects(Tab),
|
||||
ok.
|
||||
|
||||
table_info(Tab, Type) ->
|
||||
ets:info(Tab, Type).
|
||||
|
||||
foreach(Tab, Fun) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab))
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table') ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] -> %% 有可能被其他进程遍历操作删除
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key)).
|
||||
|
||||
foreach(Tab, Fun, Limit) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab), Limit)
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table', _N) ->
|
||||
ok;
|
||||
foreach_1(_Tab, _Fun, _, N) when N =< 0 ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key, N) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] ->
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key), N-1).
|
||||
|
||||
foldl(Tab, Fun, Acc) when is_function(Fun, 2) ->
|
||||
ets:foldl(Fun, Acc, Tab).
|
||||
|
||||
all_keys(Tab) ->
|
||||
MS = ets:fun2ms(fun(Rec) -> element(2, Rec) end),
|
||||
ets:select(Tab, MS).
|
||||
|
||||
tab2list(Tab) ->
|
||||
ets:tab2list(Tab).
|
||||
|
||||
add_table_copy(_Tab, _CacheNode, _TabType) ->
|
||||
ok.
|
||||
|
||||
wait_for_tables(_Tables, _TimeOut) ->
|
||||
ok.
|
||||
|
||||
close(Table) ->
|
||||
ets:delete(Table),
|
||||
IndexList = get_index_list(Table),
|
||||
delete_index_table(Table, IndexList),
|
||||
%% todo 这里没有删除persistent_term里对应的TId的值，理论上走这个逻辑已经退出app，删除的话会触发全量GC（后面再看看怎么实现更好，该存ets？）
|
||||
ok.
|
||||
|
||||
first(Table) ->
|
||||
ets:first(Table).
|
||||
|
||||
next(Table, PrevKey) ->
|
||||
ets:next(Table, PrevKey).
|
||||
|
||||
last(Table) ->
|
||||
ets:last(Table).
|
||||
|
||||
prev(Table, Key) ->
|
||||
ets:prev(Table, Key).
|
||||
|
||||
slot(TableName, Slot) ->
|
||||
ets:slot(TableName, Slot).
|
||||
|
||||
create_index_table(Table, [Pos | T]) ->
|
||||
TId = ets:new(table_index, [public, ordered_set, {keypos, 1}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
ms_cache_config:put({?MODULE, index_table_name, Table, Pos}, TId),
|
||||
create_index_table(Table, T);
|
||||
create_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
delete_index_table(Table, [Pos | T]) ->
|
||||
TId = get_index_table(Table, Pos),
|
||||
ets:delete(TId),
|
||||
delete_index_table(Table, T);
|
||||
delete_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
get_index_table(Table, Pos) ->
|
||||
ms_cache_config:get({?MODULE, index_table_name, Table, Pos}, undefined).
|
||||
|
||||
|
||||
set_index_list(Tab, IndexList) ->
|
||||
ms_cache_config:put({?MODULE, index_list, Tab}, IndexList).
|
||||
|
||||
get_index_list(Tab) ->
|
||||
ms_cache_config:get({?MODULE, index_list, Tab}, []).
|
||||
|
||||
insert_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
ets:insert(IndexTable, {{Index, RecKey}})
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
ets:delete(IndexTable, {Index, RecKey})
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
update_index(Table, OldRec, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
OldIndex = erlang:element(IndexPos, OldRec),
|
||||
NewIndex = erlang:element(IndexPos, Rec),
|
||||
case OldIndex /= NewIndex of
|
||||
true ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
%% 先插入后删除，模拟保证原子性，查询到的是删除前或者删除后的数据，不会有中间态
|
||||
ets:insert(IndexTable, {{NewIndex, RecKey}}),
|
||||
ets:delete(IndexTable, {OldIndex, RecKey});
|
||||
false ->
|
||||
skip
|
||||
end
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index_all_objects(Table) ->
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
ets:delete_all_objects(IndexTable)
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
index_read_keys(Table, IxKey, Pos) ->
|
||||
IndexTable = get_index_table(Table, Pos),
|
||||
Pat = [{{{IxKey, '$1'}}, [], ['$1']}],
|
||||
ets:select(IndexTable, Pat).
|
||||
@@ -0,0 +1,300 @@
|
||||
%%% -------------------------------------------------------------------
|
||||
%%% Author : wangyalun
|
||||
%%% Description : ets缓存数据库
|
||||
%%% Created : 2019-01-14
|
||||
%%% -------------------------------------------------------------------
|
||||
-module(ms_cache_edb2).
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-include_lib("stdlib/include/ms_transform.hrl").
|
||||
|
||||
-define(TAB_DEFAULT_OPTS, [{ram_copies, [node()]}]).
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
-export([
|
||||
init/0
|
||||
, lookup/2
|
||||
, index_read/3
|
||||
, index_match_object/5
|
||||
, lookup_element/3
|
||||
, insert/2
|
||||
, insert_new/2
|
||||
, delete/2
|
||||
, create_table/2
|
||||
%% , create_table/3
|
||||
, delete_all_objects/1
|
||||
, table_info/2
|
||||
, foreach/2
|
||||
, foreach/3
|
||||
, foldl/3
|
||||
, all_keys/1
|
||||
, tab2list/1
|
||||
, add_table_copy/3
|
||||
, wait_for_tables/2
|
||||
, member/2
|
||||
, close/1
|
||||
, first/1
|
||||
, next/2
|
||||
, slot/2
|
||||
]).
|
||||
|
||||
init() ->
|
||||
ok.
|
||||
|
||||
lookup(Tab, Key) ->
|
||||
ets:lookup(Tab, Key).
|
||||
|
||||
index_read(Table, Index, Pos) ->
|
||||
IndexTable = get_index_table(Table, Pos),
|
||||
%% RecKeys = safe_lookup_element(IndexTable, {Pos, Index}, 2),
|
||||
RecKeys = ets:lookup(IndexTable, Index),
|
||||
lists:foldl(
|
||||
fun({_, RecKey}, Acc) ->
|
||||
case ets:lookup(Table, RecKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
[Rec] ->
|
||||
[Rec | Acc]
|
||||
end
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
index_match_object(Table, Index, Pos, _Pattern, Match) ->
|
||||
IndexTable = get_index_table(Table, Pos),
|
||||
%% RecKeys = safe_lookup_element(IndexTable, {Pos, Index}, 2),
|
||||
RecKeys = ets:lookup(IndexTable, Index),
|
||||
lists:foldl(
|
||||
fun({_, RecKey}, Acc) ->
|
||||
case ets:lookup(Table, RecKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
[Rec] ->
|
||||
case lists:all(fun({P, K}) -> erlang:element(P, Rec) == K end, Match) of
|
||||
true ->
|
||||
[Rec | Acc];
|
||||
false ->
|
||||
Acc
|
||||
end
|
||||
end
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
lookup_element(Table, Key, Pos) ->
|
||||
ets:lookup_element(Table, Key, Pos).
|
||||
|
||||
insert(Tab, Record) ->
|
||||
Key = erlang:element(2, Record),
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
ets:insert(Tab, Record),
|
||||
insert_index(Tab, Record);
|
||||
[OldRecord] ->
|
||||
ets:insert(Tab, Record),
|
||||
update_index(Tab, OldRecord, Record)
|
||||
end,
|
||||
true.
|
||||
|
||||
insert_new(Tab, Record) ->
|
||||
case ets:insert_new(Tab, Record) of
|
||||
true ->
|
||||
insert_index(Tab, Record),
|
||||
true;
|
||||
false ->
|
||||
false
|
||||
end.
|
||||
|
||||
delete(Tab, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
skip;
|
||||
[Rec] ->
|
||||
ets:delete(Tab, Key),
|
||||
delete_index(Tab, Rec)
|
||||
end,
|
||||
ok.
|
||||
|
||||
|
||||
member(Tab, Key) ->
|
||||
ets:member(Tab, Key).
|
||||
|
||||
%%create_table(Tab, _Opts) ->
|
||||
%% ets:new(Tab, [public, named_table, set, {keypos, 2}, {read_concurrency, true}]).
|
||||
|
||||
create_table(Tab, Opts) ->
|
||||
TableType = proplists:get_value(type, Opts, set),
|
||||
IndexList1 = proplists:get_value(index, Opts, []),
|
||||
ets:new(Tab, [public, named_table, TableType, {keypos, 2}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
%% IndexTable = get_index_table(Tab),
|
||||
%% ets:new(IndexTable, [public, named_table, bag, {keypos, 1}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
create_index_table(Tab, IndexList1),
|
||||
set_index_list(Tab, IndexList1),
|
||||
ok.
|
||||
|
||||
delete_all_objects(Tab) ->
|
||||
ets:delete_all_objects(Tab),
|
||||
delete_index_all_objects(Tab).
|
||||
|
||||
table_info(Tab, Type) ->
|
||||
ets:info(Tab, Type).
|
||||
|
||||
foreach(Tab, Fun) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab))
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table') ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] -> %% 有可能被其他进程遍历操作删除
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key)).
|
||||
|
||||
foreach(Tab, Fun, Limit) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab), Limit)
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table', _N) ->
|
||||
ok;
|
||||
foreach_1(_Tab, _Fun, _, N) when N =< 0 ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key, N) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] ->
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key), N-1).
|
||||
|
||||
foldl(Tab, Fun, Acc) when is_function(Fun, 2) ->
|
||||
ets:foldl(Fun, Acc, Tab).
|
||||
|
||||
all_keys(Tab) ->
|
||||
MS = ets:fun2ms(fun(Rec) -> element(2, Rec) end),
|
||||
ets:select(Tab, MS).
|
||||
|
||||
tab2list(Tab) ->
|
||||
ets:tab2list(Tab).
|
||||
|
||||
add_table_copy(_Tab, _CacheNode, _TabType) ->
|
||||
ok.
|
||||
|
||||
wait_for_tables(_Tables, _TimeOut) ->
|
||||
ok.
|
||||
|
||||
close(Table) ->
|
||||
ets:delete(Table),
|
||||
IndexList = get_index_list(Table),
|
||||
delete_index_table(Table, IndexList),
|
||||
ok.
|
||||
|
||||
first(Table) ->
|
||||
ets:first(Table).
|
||||
|
||||
next(Table, PrevKey) ->
|
||||
ets:next(Table, PrevKey).
|
||||
|
||||
slot(TableName, Slot) ->
|
||||
ets:slot(TableName, Slot).
|
||||
|
||||
create_index_table(Table, [Pos | T]) ->
|
||||
TId = ets:new(table_index, [public, bag, {keypos, 1}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
persistent_term:put({?MODULE, index_table_name, Table, Pos}, TId),
|
||||
create_index_table(Table, T);
|
||||
create_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
delete_index_table(Table, [Pos | T]) ->
|
||||
TId = get_index_table(Table, Pos),
|
||||
ets:delete(TId),
|
||||
delete_index_table(Table, T);
|
||||
delete_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
get_index_table(Table, Pos) ->
|
||||
persistent_term:get({?MODULE, index_table_name, Table, Pos}, undefined).
|
||||
|
||||
set_index_list(Tab, IndexList) ->
|
||||
persistent_term:put({?MODULE, index_list, Tab}, IndexList).
|
||||
|
||||
get_index_list(Tab) ->
|
||||
persistent_term:get({?MODULE, index_list, Tab}, []).
|
||||
|
||||
insert_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
ets:insert(IndexTable, {Index, RecKey})
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
ets:delete_object(IndexTable, {Index, RecKey})
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
update_index(Table, OldRec, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
OldIndex = erlang:element(IndexPos, OldRec),
|
||||
NewIndex = erlang:element(IndexPos, Rec),
|
||||
case OldIndex /= NewIndex of
|
||||
true ->
|
||||
%% 先插入后删除，模拟保证原子性，查询到的是删除前或者删除后的数据，不会有中间态
|
||||
ets:insert(IndexTable, {NewIndex, RecKey}),
|
||||
ets:delete_object(IndexTable, {OldIndex, RecKey});
|
||||
false ->
|
||||
skip
|
||||
end
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index_all_objects(Table) ->
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
ets:delete_all_objects(IndexTable)
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
%%safe_lookup_element(Table, Key, Pos) ->
|
||||
%% try
|
||||
%% ets:lookup_element(Table, Key, Pos)
|
||||
%% catch _:_ ->
|
||||
%% []
|
||||
%% end.
|
||||
@@ -0,0 +1,316 @@
|
||||
%%% -------------------------------------------------------------------
|
||||
%%% Author : wangyalun
|
||||
%%% Description : ets缓存数据库
|
||||
%%% Created : 2019-01-14
|
||||
%%% -------------------------------------------------------------------
|
||||
-module(ms_cache_edb3).
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-include_lib("stdlib/include/ms_transform.hrl").
|
||||
|
||||
-define(TAB_DEFAULT_OPTS, [{ram_copies, [node()]}]).
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
-export([
|
||||
init/0
|
||||
, lookup/2
|
||||
, index_read/3
|
||||
, index_match_object/5
|
||||
, lookup_element/3
|
||||
, insert/2
|
||||
, insert_new/2
|
||||
, delete/2
|
||||
, create_table/2
|
||||
%% , create_table/3
|
||||
, delete_all_objects/1
|
||||
, table_info/2
|
||||
, foreach/2
|
||||
, foreach/3
|
||||
, foldl/3
|
||||
, all_keys/1
|
||||
, tab2list/1
|
||||
, add_table_copy/3
|
||||
, wait_for_tables/2
|
||||
, member/2
|
||||
, close/1
|
||||
, first/1
|
||||
, next/2
|
||||
, slot/2
|
||||
]).
|
||||
|
||||
init() ->
|
||||
ok.
|
||||
|
||||
lookup(Tab, Key) ->
|
||||
ets:lookup(Tab, Key).
|
||||
|
||||
index_read(Table, Index, Pos) ->
|
||||
IndexTable = get_index_table(Table, Pos),
|
||||
RecKeys = safe_lookup_element(IndexTable, Index, 2),
|
||||
lists:foldl(
|
||||
fun(RecKey, Acc) ->
|
||||
case ets:lookup(Table, RecKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
[Rec] ->
|
||||
[Rec | Acc]
|
||||
end
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
index_match_object(Table, Index, Pos, _Pattern, Match) ->
|
||||
IndexTable = get_index_table(Table, Pos),
|
||||
RecKeys = safe_lookup_element(IndexTable, Index, 2),
|
||||
lists:foldl(
|
||||
fun(RecKey, Acc) ->
|
||||
case ets:lookup(Table, RecKey) of
|
||||
[] ->
|
||||
Acc;
|
||||
[Rec] ->
|
||||
case lists:all(fun({P, K}) -> erlang:element(P, Rec) == K end, Match) of
|
||||
true ->
|
||||
[Rec | Acc];
|
||||
false ->
|
||||
Acc
|
||||
end
|
||||
end
|
||||
end, [], RecKeys
|
||||
).
|
||||
|
||||
lookup_element(Table, Key, Pos) ->
|
||||
ets:lookup_element(Table, Key, Pos).
|
||||
|
||||
insert(Tab, Record) ->
|
||||
Key = erlang:element(2, Record),
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
ets:insert(Tab, Record),
|
||||
insert_index(Tab, Record);
|
||||
[OldRecord] ->
|
||||
ets:insert(Tab, Record),
|
||||
update_index(Tab, OldRecord, Record)
|
||||
end,
|
||||
true.
|
||||
|
||||
insert_new(Tab, Record) ->
|
||||
case ets:insert_new(Tab, Record) of
|
||||
true ->
|
||||
insert_index(Tab, Record),
|
||||
true;
|
||||
false ->
|
||||
false
|
||||
end.
|
||||
|
||||
delete(Tab, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[] ->
|
||||
skip;
|
||||
[Rec] ->
|
||||
ets:delete(Tab, Key),
|
||||
delete_index(Tab, Rec)
|
||||
end,
|
||||
ok.
|
||||
|
||||
|
||||
member(Tab, Key) ->
|
||||
ets:member(Tab, Key).
|
||||
|
||||
%%create_table(Tab, _Opts) ->
|
||||
%% ets:new(Tab, [public, named_table, set, {keypos, 2}, {read_concurrency, true}]).
|
||||
|
||||
create_table(Tab, Opts) ->
|
||||
TableType = proplists:get_value(type, Opts, set),
|
||||
IndexList1 = proplists:get_value(index, Opts, []),
|
||||
ets:new(Tab, [public, named_table, TableType, {keypos, 2}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
create_index_table(Tab, IndexList1),
|
||||
set_index_list(Tab, IndexList1),
|
||||
ok.
|
||||
|
||||
delete_all_objects(Tab) ->
|
||||
ets:delete_all_objects(Tab),
|
||||
delete_index_all_objects(Tab).
|
||||
|
||||
table_info(Tab, Type) ->
|
||||
ets:info(Tab, Type).
|
||||
|
||||
foreach(Tab, Fun) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab))
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table') ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] -> %% 有可能被其他进程遍历操作删除
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key)).
|
||||
|
||||
foreach(Tab, Fun, Limit) when is_function(Fun, 1) ->
|
||||
ets:safe_fixtable(Tab, true),
|
||||
try
|
||||
foreach_1(Tab, Fun, ets:first(Tab), Limit)
|
||||
after
|
||||
ets:safe_fixtable(Tab, false)
|
||||
end,
|
||||
ok.
|
||||
|
||||
foreach_1(_Tab, _Fun, '$end_of_table', _N) ->
|
||||
ok;
|
||||
foreach_1(_Tab, _Fun, _, N) when N =< 0 ->
|
||||
ok;
|
||||
foreach_1(Tab, Fun, Key, N) ->
|
||||
case ets:lookup(Tab, Key) of
|
||||
[Element] ->
|
||||
Fun(Element);
|
||||
[] ->
|
||||
skip
|
||||
end,
|
||||
foreach_1(Tab, Fun, ets:next(Tab, Key), N-1).
|
||||
|
||||
foldl(Tab, Fun, Acc) when is_function(Fun, 2) ->
|
||||
ets:foldl(Fun, Acc, Tab).
|
||||
|
||||
all_keys(Tab) ->
|
||||
MS = ets:fun2ms(fun(Rec) -> element(2, Rec) end),
|
||||
ets:select(Tab, MS).
|
||||
|
||||
tab2list(Tab) ->
|
||||
ets:tab2list(Tab).
|
||||
|
||||
add_table_copy(_Tab, _CacheNode, _TabType) ->
|
||||
ok.
|
||||
|
||||
wait_for_tables(_Tables, _TimeOut) ->
|
||||
ok.
|
||||
|
||||
close(Table) ->
|
||||
ets:delete(Table),
|
||||
IndexList = get_index_list(Table),
|
||||
delete_index_table(Table, IndexList),
|
||||
ok.
|
||||
|
||||
first(Table) ->
|
||||
ets:first(Table).
|
||||
|
||||
next(Table, PrevKey) ->
|
||||
ets:next(Table, PrevKey).
|
||||
|
||||
slot(TableName, Slot) ->
|
||||
ets:slot(TableName, Slot).
|
||||
|
||||
create_index_table(Table, [Pos | T]) ->
|
||||
TId = ets:new(table_index, [public, ordered_set, {keypos, 1}, {read_concurrency, true}, {write_concurrency, true}]),
|
||||
persistent_term:put({?MODULE, index_table_name, Table, Pos}, TId),
|
||||
create_index_table(Table, T);
|
||||
create_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
delete_index_table(Table, [Pos | T]) ->
|
||||
TId = get_index_table(Table, Pos),
|
||||
ets:delete(TId),
|
||||
delete_index_table(Table, T);
|
||||
delete_index_table(_Table, []) ->
|
||||
?OK.
|
||||
|
||||
get_index_table(Table, Pos) ->
|
||||
persistent_term:get({?MODULE, index_table_name, Table, Pos}, undefined).
|
||||
|
||||
set_index_list(Tab, IndexList) ->
|
||||
persistent_term:put({?MODULE, index_list, Tab}, IndexList).
|
||||
|
||||
get_index_list(Tab) ->
|
||||
persistent_term:get({?MODULE, index_list, Tab}, []).
|
||||
|
||||
insert_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
case ets:lookup(IndexTable, Index) of
|
||||
[] ->
|
||||
ets:insert(IndexTable, {Index, [RecKey]});
|
||||
[{_, OldKeys}] -> %% todo 忽略重复key，业务保证
|
||||
ets:insert(IndexTable, {Index, [RecKey | OldKeys]})
|
||||
end
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index(Table, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
Index = erlang:element(IndexPos, Rec),
|
||||
case ets:lookup(IndexTable, Index) of
|
||||
[] ->
|
||||
?SKIP;
|
||||
[{_, OldKeys}] ->
|
||||
ets:insert(IndexTable, {Index, lists:delete(RecKey, OldKeys)})
|
||||
end
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
update_index(Table, OldRec, Rec) ->
|
||||
RecKey = erlang:element(2, Rec),
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
OldIndex = erlang:element(IndexPos, OldRec),
|
||||
NewIndex = erlang:element(IndexPos, Rec),
|
||||
case OldIndex /= NewIndex of
|
||||
true ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
%% 先插入后删除，模拟保证原子性，查询到的是删除前或者删除后的数据，不会有中间态
|
||||
case ets:lookup(IndexTable, NewIndex) of
|
||||
[] ->
|
||||
ets:insert(IndexTable, {NewIndex, [RecKey]});
|
||||
[{_, OldKeys}] -> %% todo 忽略重复key，业务保证
|
||||
ets:insert(IndexTable, {NewIndex, [RecKey | OldKeys]})
|
||||
end,
|
||||
case ets:lookup(IndexTable, OldIndex) of
|
||||
[] ->
|
||||
?SKIP;
|
||||
[{_, OldKeys1}] ->
|
||||
ets:insert(IndexTable, {OldIndex, lists:delete(RecKey, OldKeys1)})
|
||||
end;
|
||||
false ->
|
||||
?SKIP
|
||||
end
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
delete_index_all_objects(Table) ->
|
||||
IndexList = get_index_list(Table),
|
||||
lists:foreach(
|
||||
fun(IndexPos) ->
|
||||
IndexTable = get_index_table(Table, IndexPos),
|
||||
ets:delete_all_objects(IndexTable)
|
||||
end, IndexList
|
||||
),
|
||||
ok.
|
||||
|
||||
safe_lookup_element(Table, Key, Pos) ->
|
||||
try
|
||||
ets:lookup_element(Table, Key, Pos)
|
||||
catch _:_ ->
|
||||
[]
|
||||
end.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
-module(ms_cache_gs_sup).
|
||||
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% API
|
||||
-export([start_link/0]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
%% Helper macro for declaring children of supervisor
|
||||
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, infinity, Type, [I]}).
|
||||
|
||||
%% ===================================================================
|
||||
%% API functions
|
||||
%% ===================================================================
|
||||
|
||||
start_link() ->
|
||||
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
|
||||
|
||||
%% ===================================================================
|
||||
%% Supervisor callbacks
|
||||
%% ===================================================================
|
||||
|
||||
init([]) ->
|
||||
{
|
||||
ok,
|
||||
{
|
||||
{simple_one_for_one, 5, 10},
|
||||
[?CHILD(ms_cache_gs_table_sup, supervisor)]
|
||||
}
|
||||
}.
|
||||
@@ -0,0 +1,32 @@
|
||||
-module(ms_cache_gs_table_sup).
|
||||
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% API
|
||||
-export([start_link/1]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
%% Helper macro for declaring children of supervisor
|
||||
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, infinity, Type, [I]}).
|
||||
|
||||
%% ===================================================================
|
||||
%% API functions
|
||||
%% ===================================================================
|
||||
|
||||
start_link(SupName) ->
|
||||
supervisor:start_link({local, SupName}, ?MODULE, []).
|
||||
|
||||
%% ===================================================================
|
||||
%% Supervisor callbacks
|
||||
%% ===================================================================
|
||||
|
||||
init([]) ->
|
||||
{
|
||||
ok,
|
||||
{
|
||||
{simple_one_for_one, 5, 10},
|
||||
[?CHILD(ms_cache_gs, worker)]
|
||||
}
|
||||
}.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,287 @@
|
||||
%%% -------------------------------------------------------------------
|
||||
%%% Author : wangyalun
|
||||
%%% Description : 缓存mdb方法 todo 目前已经弃用
|
||||
%%% Created : 2019-01-14
|
||||
%%% -------------------------------------------------------------------
|
||||
-module(ms_cache_mdb).
|
||||
-include("base.hrl").
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%% ====================================================================
|
||||
%% Include files
|
||||
%% ====================================================================
|
||||
-define(TAB_DEFAULT_OPTS, [{ram_copies, []}]).
|
||||
|
||||
%% ====================================================================
|
||||
%% External exports
|
||||
%% ====================================================================
|
||||
-export([
|
||||
init/0
|
||||
, lookup/2
|
||||
, index_read_keys/3
|
||||
, index_read/3
|
||||
, index_match_object/5
|
||||
, lookup_element/3
|
||||
, update_element/3
|
||||
, insert/2
|
||||
, insert_new/2
|
||||
, delete/2
|
||||
, create_table/2
|
||||
%% , create_table/3
|
||||
, delete_all_objects/1
|
||||
, table_info/2
|
||||
, foreach/2
|
||||
, foreach/3
|
||||
, foldl/3
|
||||
, all_keys/1
|
||||
, tab2list/1
|
||||
, add_table_copy/3
|
||||
, wait_for_tables/2
|
||||
, member/2
|
||||
, is_table_exist/1
|
||||
, close/1
|
||||
, first/1
|
||||
, next/2
|
||||
, last/1
|
||||
, prev/2
|
||||
, slot/2
|
||||
]).
|
||||
|
||||
init() ->
|
||||
mnesia:stop(),
|
||||
mnesia:start(),
|
||||
ok.
|
||||
|
||||
lookup(Table, Key) ->
|
||||
mnesia:dirty_read(Table, Key).
|
||||
|
||||
index_read_keys(Table, Key, Pos) ->
|
||||
[element(2, Record) || Record <- mnesia:dirty_index_read(Table, Key, Pos)].
|
||||
|
||||
index_read(Table, Key, Pos) ->
|
||||
mnesia:dirty_index_read(Table, Key, Pos).
|
||||
|
||||
index_match_object(Table, _Index, Pos, Pattern, _Match) ->
|
||||
mnesia:dirty_index_match_object(Table, Pattern, Pos).
|
||||
|
||||
lookup_element(Table, Key, Pos) ->
|
||||
[Record] = mnesia:dirty_read(Table, Key),
|
||||
erlang:element(Pos, Record).
|
||||
|
||||
update_element(Table, Key, {Pos, Value}) ->
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[Record] ->
|
||||
NewRecord = erlang:setelement(Pos, Record, Value),
|
||||
mnesia:dirty_write(Table, NewRecord),
|
||||
true;
|
||||
_ ->
|
||||
false
|
||||
end;
|
||||
update_element(Table, Key, PosValueList) ->
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[Record] ->
|
||||
NewRecord = lists:foldl(fun(Pos, Value, Acc) -> erlang:setelement(Pos, Acc, Value) end, Record, PosValueList),
|
||||
mnesia:dirty_write(Table, NewRecord),
|
||||
true;
|
||||
_ ->
|
||||
false
|
||||
end.
|
||||
|
||||
insert(Table, Record) ->
|
||||
mnesia:dirty_write(Table, Record).
|
||||
|
||||
insert_new(Table, Record) ->
|
||||
Key = erlang:element(2, Record),
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[] ->
|
||||
mnesia:dirty_write(Table, Record),
|
||||
true;
|
||||
[_] ->
|
||||
false
|
||||
end.
|
||||
|
||||
delete(Table, Key) ->
|
||||
mnesia:dirty_delete(Table, Key).
|
||||
|
||||
member(Table, Key) ->
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[] ->
|
||||
false;
|
||||
[_] ->
|
||||
true
|
||||
end.
|
||||
|
||||
%% @doc
|
||||
%% 创建一张mnesia表
|
||||
%%
|
||||
-spec create_table(TableName :: atom(), Options :: list()) -> ok.
|
||||
create_table(TableName, Options1) ->
|
||||
%% IndexList =
|
||||
%% lists:usort([
|
||||
%% begin
|
||||
%% case erlang:is_tuple(Index) of
|
||||
%% true ->
|
||||
%% erlang:element(1, Index);
|
||||
%% false ->
|
||||
%% Index
|
||||
%% end
|
||||
%% end || Index <- Handler:index_list()
|
||||
%% ]),
|
||||
%%
|
||||
%% IndexList1 =
|
||||
%% case Handler:key_list() of
|
||||
%% [FirstKey | _] ->
|
||||
%% [FirstKey | lists:delete(FirstKey, IndexList)];
|
||||
%% _ ->
|
||||
%% IndexList
|
||||
%% end,
|
||||
%% RecName = Handler:record_name(),
|
||||
%% DefaultOptions = [{type, set}, {attributes, Handler:field_list()}, {record_name, RecName}, {index, IndexList1},
|
||||
%% {ram_copies, [erlang:node()]},
|
||||
%% {storage_properties, [{ets, [{read_concurrency,true}, {write_concurrency,true}]}]}
|
||||
%% ],
|
||||
%% Options1 =
|
||||
%% lists:foldl(
|
||||
%% fun({Key, Val}, Acc) ->
|
||||
%% lists:keystore(Key, 1, Acc, {Key, Val})
|
||||
%% end, DefaultOptions, Options),
|
||||
|
||||
{atomic, ok} = mnesia:create_table(TableName, Options1),
|
||||
?OK.
|
||||
|
||||
%%create_table(Table, Opts) ->
|
||||
%% {ok, NewOpts} = merge_options(?TAB_DEFAULT_OPTS, Opts),
|
||||
%% {atomic, ok} = mnesia:create_table(Table, NewOpts),
|
||||
%% ok.
|
||||
|
||||
|
||||
delete_all_objects(Table) ->
|
||||
mnesia:clear_table(Table).
|
||||
|
||||
table_info(Table, Type) ->
|
||||
mnesia:table_info(Table, Type).
|
||||
|
||||
foreach(Table, Fun) when is_function(Fun, 1) ->
|
||||
safe_fixtable(Table, ?TRUE),
|
||||
try
|
||||
foreach_1(Table, Fun, mnesia:dirty_first(Table))
|
||||
after
|
||||
safe_fixtable(Table, ?FALSE)
|
||||
end.
|
||||
|
||||
foreach_1(_Table, _Fun, '$end_of_table') ->
|
||||
ok;
|
||||
foreach_1(Table, Fun, Key) ->
|
||||
NextKey = mnesia:dirty_next(Table, Key),
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[Element] ->
|
||||
Fun(Element);
|
||||
[] -> %% 有可能被其他进程遍历操作删除了
|
||||
?SKIP
|
||||
end,
|
||||
foreach_1(Table, Fun, NextKey).
|
||||
|
||||
foreach(Table, Fun, Limit) when is_function(Fun, 1) ->
|
||||
safe_fixtable(Table, ?TRUE),
|
||||
try
|
||||
foreach_1(Table, Fun, mnesia:dirty_first(Table), Limit)
|
||||
after
|
||||
safe_fixtable(Table, ?FALSE)
|
||||
end.
|
||||
|
||||
foreach_1(_Table, _Fun, '$end_of_table', _N) ->
|
||||
ok;
|
||||
foreach_1(_Table, _Fun, _, N) when N =< 0 ->
|
||||
ok;
|
||||
foreach_1(Table, Fun, Key, N) ->
|
||||
NextKey = mnesia:dirty_next(Table, Key),
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[Element] ->
|
||||
Fun(Element);
|
||||
[] -> %% 有可能被其他进程遍历操作删除了
|
||||
?SKIP
|
||||
end,
|
||||
foreach_1(Table, Fun, NextKey, N-1).
|
||||
|
||||
foldl(Table, Fun, Acc) when is_function(Fun, 2) ->
|
||||
safe_fixtable(Table, ?TRUE),
|
||||
try
|
||||
foldl(Table, Fun, Acc, mnesia:dirty_first(Table))
|
||||
after
|
||||
safe_fixtable(Table, ?FALSE)
|
||||
end.
|
||||
|
||||
foldl(_Table, _Fun, Acc, '$end_of_table') ->
|
||||
Acc;
|
||||
foldl(Table, Fun, Acc, Key) ->
|
||||
NextKey = mnesia:dirty_next(Table, Key),
|
||||
foldl(Table, Fun, lists:foldl(Fun, Acc, mnesia:dirty_read(Table, Key)), NextKey).
|
||||
|
||||
|
||||
all_keys(Table) ->
|
||||
mnesia:dirty_all_keys(Table).
|
||||
|
||||
tab2list(Table) ->
|
||||
tab2list_1(Table, mnesia:dirty_first(Table), []).
|
||||
|
||||
tab2list_1(_Table, '$end_of_table', Acc) ->
|
||||
Acc;
|
||||
tab2list_1(Table, Key, Acc) ->
|
||||
case mnesia:dirty_read(Table, Key) of
|
||||
[Rec] ->
|
||||
tab2list_1(Table, mnesia:dirty_next(Table, Key), [Rec | Acc]);
|
||||
[] -> %% 有可能被其他进程遍历操作删除了
|
||||
tab2list_1(Table, mnesia:dirty_next(Table, Key), Acc)
|
||||
end.
|
||||
|
||||
add_table_copy(Table, CacheNode, TableType) ->
|
||||
mnesia:add_table_copy(Table, CacheNode, TableType).
|
||||
|
||||
wait_for_tables(Tables, TimeOut) ->
|
||||
mnesia:wait_for_tables(Tables, TimeOut).
|
||||
|
||||
close(Table) ->
|
||||
mnesia:delete_table(Table),
|
||||
ok.
|
||||
|
||||
%%merge_options([{ram_copies, _} = Opt | T], Acc) ->
|
||||
%% case lists:keymember(ram_copies, 1, Acc) orelse
|
||||
%% lists:keymember(disc_copies, 1, Acc) orelse
|
||||
%% lists:keymember(disc_only_copies, 1, Acc) of
|
||||
%% true ->
|
||||
%% merge_options(T, Acc);
|
||||
%% false ->
|
||||
%% merge_options(T, [Opt | Acc])
|
||||
%% end;
|
||||
%%merge_options([Opt | T], Acc) ->
|
||||
%% case lists:keymember(element(1, Opt), 1, Acc) of
|
||||
%% true ->
|
||||
%% merge_options(T, Acc);
|
||||
%% false ->
|
||||
%% merge_options(T, [Opt | Acc])
|
||||
%% end;
|
||||
%%merge_options([], Acc) ->
|
||||
%% {ok, Acc}.
|
||||
|
||||
is_table_exist(Table) ->
|
||||
Tables = mnesia:system_info(tables),
|
||||
lists:member(Table, Tables).
|
||||
|
||||
safe_fixtable(Table, Bool) ->
|
||||
?DEBUG(?MODULE, "StorageType = ~p, Table = ~p, Bool = ~p", [mnesia:table_info(Table, storage_type), Table, Bool]),
|
||||
mnesia_lib:db_fixtable(mnesia:table_info(Table, storage_type), Table, Bool).
|
||||
|
||||
first(Table) ->
|
||||
mnesia:dirty_first(Table).
|
||||
|
||||
next(Table, PrevKey) ->
|
||||
mnesia:dirty_next(Table, PrevKey).
|
||||
|
||||
last(Table) ->
|
||||
mnesia:dirty_last(Table).
|
||||
|
||||
prev(Table, NextKey) ->
|
||||
mnesia:dirty_prev(Table, NextKey).
|
||||
|
||||
slot(Table, Slot) ->
|
||||
mnesia:dirty_slot(Table, Slot).
|
||||
@@ -0,0 +1,183 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2023, <COMPANY>
|
||||
%%% @doc
|
||||
%%% 补丁进程
|
||||
%%% @end
|
||||
%%% Created : 08. 9月 2023 15:33
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_patch_lib).
|
||||
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
-include("base.hrl").
|
||||
-export([patch/2]).
|
||||
|
||||
-define(TABLE_PATCH_VERSION, '$patch_version$').
|
||||
|
||||
%%%===================================================================
|
||||
%%% API
|
||||
%%%===================================================================
|
||||
%% 执行补丁程序
|
||||
patch(DbPool, SyncOptions) ->
|
||||
case is_initialized(DbPool) of
|
||||
true ->
|
||||
first_init(DbPool, SyncOptions);
|
||||
false ->
|
||||
case SyncOptions of
|
||||
#{patch_run_mod := auto} ->
|
||||
ok = run_patch(DbPool, SyncOptions);
|
||||
#{patch_run_mod := {env, Evn}} ->
|
||||
?IF(proplists:is_defined(Evn, os:env()), run_patch(DbPool, SyncOptions), ok);
|
||||
#{} ->
|
||||
ok
|
||||
end
|
||||
end.
|
||||
|
||||
%%%===================================================================
|
||||
%%% Internal functions
|
||||
%%%===================================================================
|
||||
%% 是否已初始化
|
||||
is_initialized(DbPool) ->
|
||||
SQL = "SHOW TABLES;",
|
||||
{ok, _Column, Tables} = ms_cache:execute(DbPool, SQL),
|
||||
not lists:member([erlang:atom_to_binary(?TABLE_PATCH_VERSION)], Tables).
|
||||
|
||||
%% 首次初始化
|
||||
first_init(DbPool, SyncOptions) ->
|
||||
case maps:get(patch_prefix, SyncOptions, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
ok;
|
||||
PatchPrefix ->
|
||||
init_patch_version(DbPool),
|
||||
insert_init_data(DbPool, PatchPrefix)
|
||||
end.
|
||||
|
||||
%% 初始化打补丁相关程序
|
||||
init_patch_version(DbPool) ->
|
||||
SQL = io_lib:format("CREATE TABLE IF NOT EXISTS `~s` (
|
||||
`version` int(11) NOT NULL DEFAULT '0',
|
||||
`func_list` text NOT NULL,
|
||||
PRIMARY KEY (`version`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;", [?TABLE_PATCH_VERSION]),
|
||||
ok = mysql_poolboy:query(DbPool, SQL).
|
||||
|
||||
%% 插入初始数据
|
||||
insert_init_data(DbPool, PatchPrefix) ->
|
||||
case get_mod_list(PatchPrefix, 0) of
|
||||
[] ->
|
||||
ok;
|
||||
L ->
|
||||
{Version, Mod} = lists:last(L),
|
||||
SQL = io_lib:format("INSERT INTO `~s` (`version`, `func_list`) VALUES(?, ?);", [?TABLE_PATCH_VERSION]),
|
||||
Exports = proplists:delete(module_info, erlang:apply(Mod, module_info, [exports])),
|
||||
ok = mysql_poolboy:query(DbPool, SQL, [Version, term_to_string(Exports)])
|
||||
end.
|
||||
|
||||
%% 执行补丁程序
|
||||
run_patch(DbPool, SyncOptions) ->
|
||||
case maps:get(patch_prefix, SyncOptions, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
ok;
|
||||
PatchPrefix ->
|
||||
Limit = maps:get(limit, SyncOptions, 1),
|
||||
{LatestVersion, Maps} = load_patch_version_table(DbPool, Limit),
|
||||
VersionModList = get_mod_list(PatchPrefix, LatestVersion),
|
||||
do_patch(DbPool, VersionModList, Maps)
|
||||
end,
|
||||
ok.
|
||||
|
||||
%% 载入表数据
|
||||
load_patch_version_table(DbPool, Limit) ->
|
||||
SQL = io_lib:format("SELECT * FROM `~s` ORDER BY `version` DESC LIMIT ?;", [?TABLE_PATCH_VERSION]),
|
||||
case mysql_poolboy:query(DbPool, SQL, [Limit]) of
|
||||
{ok, _Column, []} ->
|
||||
{0, #{}};
|
||||
{ok, _Column, Rows} ->
|
||||
[LatestVersion | _] = lists:last(Rows),
|
||||
L = [{Version, string_to_term(FuncList)} || [Version, FuncList] <- Rows],
|
||||
{LatestVersion, maps:from_list(L)}
|
||||
end.
|
||||
|
||||
%% 获取补丁模块名
|
||||
get_mod_list(PatchPrefix, LatestVersion) ->
|
||||
case application:get_application() of
|
||||
{ok, AppName} ->
|
||||
{ok, Pattern} = re:compile(PatchPrefix ++ "([0-9]{8})"),
|
||||
L = lists:filtermap(
|
||||
fun(Filename) ->
|
||||
Basename = filename:basename(Filename, ".beam"),
|
||||
case re:run(Basename, Pattern, [{capture, all_but_first, list}]) of
|
||||
{match, [StrVersion]} ->
|
||||
Version = erlang:list_to_integer(StrVersion),
|
||||
?IF(Version >= LatestVersion, {true, {Version, erlang:list_to_atom(Basename)}}, false);
|
||||
nomatch ->
|
||||
false
|
||||
end
|
||||
end, filelib:wildcard("*.beam", code:lib_dir(AppName) ++ "/ebin")),
|
||||
lists:sort(L);
|
||||
undefined ->
|
||||
[]
|
||||
end.
|
||||
|
||||
%% 执行打补丁
|
||||
do_patch(DbPool, [{Version, Mod} | T], Maps) ->
|
||||
Exports = proplists:delete(module_info, erlang:apply(Mod, module_info, [exports])),
|
||||
case Maps of
|
||||
#{Version := FuncList} ->
|
||||
case Exports -- FuncList of
|
||||
[] ->
|
||||
ok;
|
||||
ExecuteFuncList ->
|
||||
?INFO(?MODULE, "append execute patch version:~p total number of functions:~p~n", [Version, erlang:length(ExecuteFuncList)]),
|
||||
do_patch_1(DbPool, Version, Mod, ExecuteFuncList)
|
||||
end;
|
||||
#{} ->
|
||||
?INFO(?MODULE, "execute patch version:~p total number of functions:~p~n", [Version, erlang:length(Exports)]),
|
||||
do_patch_1(DbPool, Version, Mod, Exports)
|
||||
end,
|
||||
do_patch(DbPool, T, Maps);
|
||||
do_patch(_DbPool, [], _Maps) ->
|
||||
ok.
|
||||
|
||||
do_patch_1(DbPool, Version, Mod, [{Fun, Arity} | T]) ->
|
||||
try
|
||||
?INFO(?MODULE, "execute patch ~w:~w/~w", [Mod, Fun, Arity]),
|
||||
{Time, _} = timer:tc(erlang, apply, [Mod, Fun, [DbPool]]),
|
||||
?INFO(?MODULE, "execute patch ~w:~w/~w finished! elapsed time:~.2f/s", [Mod, Fun, Arity, Time / 1000 / 1000]),
|
||||
update_patch_version_rec(DbPool, Version, {Fun, Arity}),
|
||||
do_patch_1(DbPool, Version, Mod, T)
|
||||
catch
|
||||
Error:Reason:Trace ->
|
||||
?ERROR(?MODULE, "Error:~p Reason:~p~nTrace:~tp~n", [Trace, Error, Reason]),
|
||||
erlang:error(Reason)
|
||||
end;
|
||||
do_patch_1(_DbPool, _Version, _Mod, []) ->
|
||||
ok.
|
||||
|
||||
%% 更新版本记录
|
||||
update_patch_version_rec(DbPool, Version, Func) ->
|
||||
SelectSQL = io_lib:format("SELECT `func_list` FROM `~s` WHERE `version`= ?;", [?TABLE_PATCH_VERSION]),
|
||||
case mysql_poolboy:query(DbPool, SelectSQL, [Version]) of
|
||||
{ok, _Column, []} ->
|
||||
UpFuncList = [Func];
|
||||
{ok, _Column, [[FuncListStr]]} ->
|
||||
UpFuncList = string_to_term(FuncListStr) ++ [Func]
|
||||
end,
|
||||
ReplaceSQL = io_lib:format("REPLACE INTO `~s` (`version`,`func_list`) VALUES (?, ?);", [?TABLE_PATCH_VERSION]),
|
||||
ok = mysql_poolboy:query(DbPool, ReplaceSQL, [Version, term_to_string(UpFuncList)]).
|
||||
|
||||
string_to_term(Str) when is_binary(Str) ->
|
||||
string_to_term(unicode:characters_to_list(Str));
|
||||
string_to_term(Str) when is_list(Str) ->
|
||||
case erl_scan:string(Str ++ ".") of
|
||||
{ok, Tokens, _} ->
|
||||
case erl_parse:parse_term(Tokens) of
|
||||
{ok, Term} -> Term;
|
||||
_Err -> undefined
|
||||
end;
|
||||
Error ->
|
||||
throw({error, Error})
|
||||
end.
|
||||
|
||||
term_to_string(Term) ->
|
||||
lists:flatten(io_lib:format("~w", [Term])).
|
||||
@@ -0,0 +1,38 @@
|
||||
|
||||
-module(ms_cache_sup).
|
||||
|
||||
-behaviour(supervisor).
|
||||
|
||||
%% API
|
||||
-export([start_link/0]).
|
||||
|
||||
%% Supervisor callbacks
|
||||
-export([init/1]).
|
||||
|
||||
%% Helper macro for declaring children of supervisor
|
||||
-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, infinity, Type, [I]}).
|
||||
|
||||
%% ===================================================================
|
||||
%% API functions
|
||||
%% ===================================================================
|
||||
|
||||
start_link() ->
|
||||
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
|
||||
|
||||
%% ===================================================================
|
||||
%% Supervisor callbacks
|
||||
%% ===================================================================
|
||||
|
||||
init([]) ->
|
||||
{
|
||||
ok,
|
||||
{
|
||||
{one_for_one, 5, 10},
|
||||
[
|
||||
%% ?CHILD(ms_cache_dao_gs_sup, supervisor),
|
||||
?CHILD(ms_cache_dao_sync_gs_sup, supervisor),
|
||||
?CHILD(ms_cache_dao_merge_gs_sup, supervisor),
|
||||
?CHILD(ms_cache_gs_sup, supervisor)
|
||||
]
|
||||
}
|
||||
}.
|
||||
@@ -0,0 +1,185 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2025, <COMPANY>
|
||||
%%% @doc
|
||||
%%%
|
||||
%%% @end
|
||||
%%% Created : 09. 1月 2025 21:18
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_tkv).
|
||||
%% API
|
||||
-export([
|
||||
init/0
|
||||
, init/1
|
||||
, is_init/1
|
||||
, get/2
|
||||
, get/3
|
||||
, add/2
|
||||
, add/3
|
||||
, update/2
|
||||
, update/3
|
||||
, replace/2
|
||||
, replace/3
|
||||
, delete/2
|
||||
, set_kv_list/2
|
||||
, get_key_list/1
|
||||
, get_value_list/1
|
||||
, get_kv_list/1
|
||||
, get_map/1
|
||||
, get_size/1
|
||||
, delete/1
|
||||
, is_key/2
|
||||
, foreach/2
|
||||
, fold/3
|
||||
]).
|
||||
|
||||
init() ->
|
||||
ok.
|
||||
|
||||
init(_) ->
|
||||
ok.
|
||||
|
||||
is_init(Type) ->
|
||||
erlang:get({?MODULE, Type}) /= undefined.
|
||||
|
||||
%% 添加一个record数据
|
||||
-spec(add(Type::atom(), Value::term()) -> term()).
|
||||
add(Type, Value) when is_tuple(Value) ->
|
||||
add(Type, erlang:element(2, Value), Value).
|
||||
|
||||
%% 添加数据
|
||||
-spec(add(Type :: atom(), Key :: term(), Value :: term()) -> term()).
|
||||
add(Type, Key, Value) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
erlang:put({?MODULE, Type}, #{Key => Value});
|
||||
MapValue ->
|
||||
erlang:put({?MODULE, Type}, maps:put(Key, Value, MapValue))
|
||||
end.
|
||||
|
||||
%% 获取某个值
|
||||
-spec(get(Type :: atom(), Key :: term()) -> term() | undefined).
|
||||
get(Type, Key) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
undefined;
|
||||
MapValue ->
|
||||
maps:get(Key, MapValue, undefined)
|
||||
end.
|
||||
%% 获取某个值
|
||||
-spec(get(Type :: atom(), Key :: term(), Default::term()) -> term() | Default::term()).
|
||||
get(Type, Key, Default) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
Default;
|
||||
MapValue ->
|
||||
maps:get(Key, MapValue, Default)
|
||||
end.
|
||||
|
||||
set_kv_list(Type, KVList) ->
|
||||
MapValue = maps:from_list(KVList),
|
||||
erlang:put({?MODULE, Type}, MapValue),
|
||||
ok.
|
||||
|
||||
get_kv_list(Type) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
[];
|
||||
MapValue ->
|
||||
maps:to_list(MapValue)
|
||||
end.
|
||||
|
||||
get_map(Type) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
#{};
|
||||
MapValue ->
|
||||
MapValue
|
||||
end.
|
||||
|
||||
get_value_list(Type) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
[];
|
||||
MapValue ->
|
||||
maps:values(MapValue)
|
||||
end.
|
||||
|
||||
get_key_list(Type) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
[];
|
||||
MapValue ->
|
||||
maps:keys(MapValue)
|
||||
end.
|
||||
|
||||
get_size(Type) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
0;
|
||||
MapValue ->
|
||||
maps:size(MapValue)
|
||||
end.
|
||||
|
||||
is_key(Type, Key) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
false;
|
||||
MapValue ->
|
||||
maps:is_key(Key, MapValue)
|
||||
end.
|
||||
|
||||
%% 删除元素
|
||||
-spec(delete(Type :: atom(), Key :: term()) -> term() | undefined).
|
||||
delete(Type, Key) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
ok;
|
||||
MapValue ->
|
||||
erlang:put({?MODULE, Type}, maps:remove(Key, MapValue))
|
||||
end.
|
||||
|
||||
delete(Type) ->
|
||||
erlang:erase({?MODULE, Type}).
|
||||
|
||||
update(Type, NewValue) when is_tuple(NewValue) ->
|
||||
update(Type, erlang:element(2, NewValue), NewValue).
|
||||
update(Type, Key, NewValue) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
error({update_type_not_exists, Type});
|
||||
MapValue ->
|
||||
case maps:is_key(Key, MapValue) of
|
||||
false ->
|
||||
error({update_key_not_exists, Type});
|
||||
true ->
|
||||
NewMap = maps:update(Key, NewValue, MapValue),
|
||||
erlang:put({?MODULE, Type}, NewMap)
|
||||
end
|
||||
end.
|
||||
|
||||
replace(Type, NewValue) when is_tuple(NewValue) ->
|
||||
replace(Type, erlang:element(2, NewValue), NewValue).
|
||||
replace(Type, Key, NewValue) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
add(Type, Key, NewValue);
|
||||
MapValue ->
|
||||
NewMap = maps:put(Key, NewValue, MapValue),
|
||||
erlang:put({?MODULE, Type}, NewMap)
|
||||
end.
|
||||
|
||||
foreach(Type, Fun) when is_function(Fun, 2) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
ok;
|
||||
MapValue ->
|
||||
maps:foreach(Fun, MapValue)
|
||||
end.
|
||||
|
||||
fold(Type, Fun, Init) when is_function(Fun, 3) ->
|
||||
case erlang:get({?MODULE, Type}) of
|
||||
undefined ->
|
||||
Init;
|
||||
MapValue ->
|
||||
maps:fold(Fun, Init, MapValue)
|
||||
end.
|
||||
@@ -0,0 +1,479 @@
|
||||
%%%-------------------------------------------------------------------
|
||||
%%% @copyright (C) 2019, <COMPANY>
|
||||
%%% @doc
|
||||
%%% ms_cache组件模块内部方法实现
|
||||
%%% @end
|
||||
%%% Created : 2019-09-03 22:01:42
|
||||
%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_util).
|
||||
%%--------------------------------------------------------------------
|
||||
%% include
|
||||
%%--------------------------------------------------------------------
|
||||
-include("ms_cache.hrl").
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% API exports
|
||||
%%--------------------------------------------------------------------
|
||||
-export([
|
||||
get_error_message/3
|
||||
, get_table_worker_count/1
|
||||
, get_table_worker_num/2
|
||||
, get_key_index/1
|
||||
, get_table_name/2
|
||||
, get_table_name/3
|
||||
, get_table_name_by_key/3
|
||||
, get_cache_db_mod/0
|
||||
, get_cache_opt_mod/1
|
||||
, get_data_node/0
|
||||
, get_all_table_workers/1
|
||||
, get_table_worker/1
|
||||
, get_table_worker/2
|
||||
, get_table_supervisor_name/1
|
||||
, get_table_worker_name/2
|
||||
, map_to_record/1
|
||||
, record_to_map/1
|
||||
, format_data/1
|
||||
, now_ms/0
|
||||
, wait_n_reply/3
|
||||
, wait_n_reply/4
|
||||
]).
|
||||
|
||||
-export([
|
||||
get_add_column_sql/2
|
||||
, get_del_column_sql/2
|
||||
, get_upd_column_sql/2
|
||||
, get_upd_column_sql/3
|
||||
]).
|
||||
|
||||
%%%===================================================================
|
||||
%%% API functions
|
||||
%%%===================================================================
|
||||
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取错误信息
|
||||
%%
|
||||
%% @spec get_error_message(Tab :: atom(), Key :: integer() | binary() | tuple(), Reason :: list()) -> ok.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_error_message(Tab :: atom(), Key :: integer() | binary() | tuple(), Reason :: list()) -> ok.
|
||||
get_error_message(Tab, Key, Reason) ->
|
||||
lists:flatten(io_lib:format(Reason ++ ", " ++ atom_to_list(Tab) ++ "-~p", [Key])).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取主键的第一个索引值
|
||||
%%
|
||||
%% @spec get_key_index(Key :: integer() | binary() | tuple()) -> ok.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_key_index(Key :: integer() | binary() | tuple()) -> ok.
|
||||
get_key_index(Key) when is_tuple(Key) ->
|
||||
element(1, Key);
|
||||
get_key_index(Key) ->
|
||||
Key.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 根据表类型、主键/第一个索引值获取表名
|
||||
%%
|
||||
%% @spec get_table_name_by_key(Tab :: atom(), Type :: atom()， Key :: integer() | binary() | tuple()) -> ok.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_name_by_key(Table :: atom(), Type :: atom(), Key :: integer() | binary() | tuple()) -> TableName :: atom().
|
||||
get_table_name_by_key(Table, Type, Key) when is_tuple(Key) ->
|
||||
get_table_name_by_key(Table, Type, element(1, Key));
|
||||
get_table_name_by_key(Table, Type, Key) ->
|
||||
WorkerCount = get_table_worker_count(Table),
|
||||
WorkerNum = erlang:phash2(Key, WorkerCount) + 1,
|
||||
get_table_name(Table, Type, WorkerNum).
|
||||
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 根据类型获取缓存表名
|
||||
%%
|
||||
%% @spec get_table_name(Tab :: atom(), Type :: atom()) -> ok.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_name(Tab :: atom(), Type :: atom()) -> ok.
|
||||
get_table_name(Tab, Type) ->
|
||||
case ms_cache_config:get({ms_cache, table_name, {Tab, Type}}, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
TableName =
|
||||
case Type of
|
||||
idx ->
|
||||
Handler = ?TABLE_HANDLER(Tab),
|
||||
KeyList = Handler:key_list(),
|
||||
case length(KeyList) < 2 of
|
||||
true ->
|
||||
none;
|
||||
false ->
|
||||
list_to_atom("ms_cache_" ++ atom_to_list(Tab) ++ "_" ++ atom_to_list(idx))
|
||||
end;
|
||||
_ ->
|
||||
list_to_atom("ms_cache_" ++ atom_to_list(Tab) ++ "_" ++ atom_to_list(Type))
|
||||
end,
|
||||
ms_cache_config:put({ms_cache, table_name, {Tab, Type}}, TableName),
|
||||
TableName;
|
||||
TableName ->
|
||||
TableName
|
||||
end.
|
||||
|
||||
get_table_name(Tab, Type, WorkerNum) ->
|
||||
case ms_cache_config:get({ms_cache, table_name, {Tab, Type, WorkerNum}}, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
TableName =
|
||||
case Type of
|
||||
idx ->
|
||||
Handler = ?TABLE_HANDLER(Tab),
|
||||
KeyList = Handler:key_list(),
|
||||
case length(KeyList) < 2 of
|
||||
true ->
|
||||
none;
|
||||
false ->
|
||||
list_to_atom("ms_cache_" ++ atom_to_list(Tab) ++ "_" ++ atom_to_list(idx) ++ "_" ++ integer_to_list(WorkerNum))
|
||||
end;
|
||||
_ ->
|
||||
list_to_atom("ms_cache_" ++ atom_to_list(Tab) ++ "_" ++ atom_to_list(Type) ++ "_" ++ integer_to_list(WorkerNum))
|
||||
end,
|
||||
ms_cache_config:put({ms_cache, table_name, {Tab, Type, WorkerNum}}, TableName),
|
||||
TableName;
|
||||
TableName ->
|
||||
TableName
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 缓存数据库类型对应的模块 ets | mnesia
|
||||
%% ets类型 =》 ms_cache_edb
|
||||
%% mnesia类型 =》 ms_cache_mdb(效率问题已被弃用，可能后续部分功能并没有太好的支持，使用的话需要多测试)
|
||||
%%
|
||||
%%
|
||||
%% @spec get_cache_db_mod() -> ms_cache_edb | ms_cache_mdb.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_cache_db_mod() -> ms_cache_edb | ms_cache_mdb.
|
||||
get_cache_db_mod() -> ms_cache_edb.
|
||||
%% case application:get_env(game_server, is_develop, ?FALSE) of
|
||||
%% ?TRUE ->
|
||||
%% ms_cache_mdb;
|
||||
%% ?FALSE ->
|
||||
%% ms_cache_edb
|
||||
%% end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 缓存记录数据操作对应的模块
|
||||
%% 非安全模式使用ets数据表存储操作 => ms_cache_edb，效率高，容灾差
|
||||
%% 安全模式使用dets数据表存储操作 => ms_cache_ddb，效率低，容灾强
|
||||
%%
|
||||
%%
|
||||
%% @spec get_cache_opt_mod(IsSafeMode :: boolean()) -> ms_cache_edb | ms_cache_ddb.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_cache_opt_mod(IsSafeMode :: boolean()) -> ms_cache_edb | ms_cache_mdb.
|
||||
get_cache_opt_mod(?TRUE) -> ms_cache_ddb;
|
||||
get_cache_opt_mod(_) -> ms_cache_edb.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取缓存节点(存在缓存节点,其他节点经由缓存节点获取数据)
|
||||
%%
|
||||
%% @spec get_data_node() -> CacheNode :: atom().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_data_node() -> CacheNode :: atom().
|
||||
get_data_node() ->
|
||||
case ms_cache_config:get_env(data_node, undefined) of
|
||||
undefined ->
|
||||
erlang:node();
|
||||
CacheNode ->
|
||||
CacheNode
|
||||
end.
|
||||
|
||||
get_table_worker_count(Tab) ->
|
||||
Handler = ?TABLE_HANDLER(Tab),
|
||||
case Handler:is_pre_load() of
|
||||
true ->
|
||||
1;
|
||||
false ->
|
||||
ms_cache_config:get_env(table_worker_count, ?WORKER_COUNTER_DEFAULT)
|
||||
end.
|
||||
|
||||
get_table_worker_num(Tab, Key) when is_tuple(Key) ->
|
||||
case get_table_worker_count(Tab) of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
erlang:phash2(element(1, Key), WorkerCount) + 1;
|
||||
_ ->
|
||||
0
|
||||
end;
|
||||
get_table_worker_num(Tab, Key) ->
|
||||
case get_table_worker_count(Tab) of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
erlang:phash2(Key, WorkerCount) + 1;
|
||||
_ ->
|
||||
0
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取数据表对应的所有缓存进程
|
||||
%%
|
||||
%% @spec get_all_table_workers(Tab :: atom()) -> AllWorkers :: list().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_all_table_workers(Tab :: atom()) -> AllWorkers :: list().
|
||||
get_all_table_workers(Tab) ->
|
||||
case get_table_worker_count(Tab) of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
lists:foldl(
|
||||
fun(WorkerNum, Acc) ->
|
||||
SrvName = get_table_worker_name(Tab, WorkerNum),
|
||||
[SrvName | Acc]
|
||||
end, [], lists:seq(1, WorkerCount)
|
||||
);
|
||||
_ ->
|
||||
[]
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 随机获取一个数据表对应的的缓存进程
|
||||
%%
|
||||
%% @spec get_table_worker(Tab :: atom()) -> Worker :: atom().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_worker(Tab :: atom()) -> Worker :: atom().
|
||||
get_table_worker(Tab) ->
|
||||
case get_table_worker_count(Tab)of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
WorkerNum = rand:uniform(WorkerCount),
|
||||
get_table_worker_name(Tab, WorkerNum);
|
||||
_ ->
|
||||
undefined
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 根据主键获取数据表对应的的缓存进程
|
||||
%%
|
||||
%% @spec get_table_worker(Tab :: atom(), Key :: integer() | binary() | tuple()) -> Worker :: atom().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_worker(Tab :: atom(), Key :: integer() | binary() | tuple()) -> Worker :: atom().
|
||||
get_table_worker(Tab, Key) when is_tuple(Key) ->
|
||||
case get_table_worker_count(Tab) of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
WorkerNum = erlang:phash2(element(1, Key), WorkerCount) + 1,
|
||||
get_table_worker_name(Tab, WorkerNum);
|
||||
_ ->
|
||||
undefined
|
||||
end;
|
||||
get_table_worker(Tab, Key) ->
|
||||
case get_table_worker_count(Tab) of
|
||||
WorkerCount when WorkerCount > 0 ->
|
||||
WorkerNum = erlang:phash2(Key, WorkerCount) + 1,
|
||||
get_table_worker_name(Tab, WorkerNum);
|
||||
_ ->
|
||||
undefined
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 根据序号获取缓存的工作进程名
|
||||
%%
|
||||
%% @spec get_table_maworker_name(Tab :: atom(), WorkerNum :: integer()) -> Worker :: atom().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_worker_name(Tab :: atom(), WorkerNum :: integer()) -> Worker :: atom().
|
||||
get_table_worker_name(Tab, WorkerNum) ->
|
||||
case ms_cache_config:get({ms_cache, table_worker_name, {Tab, WorkerNum}}, ?UNDEFINED) of
|
||||
?UNDEFINED ->
|
||||
WorkerName = list_to_atom("ms_cache_gs_" ++ atom_to_list(Tab) ++ "_" ++ integer_to_list(WorkerNum)),
|
||||
ms_cache_config:put({ms_cache, table_worker_name, {Tab, WorkerNum}}, WorkerName),
|
||||
WorkerName;
|
||||
WorkerName ->
|
||||
WorkerName
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 随机获取一个数据表缓存进程对应的监控进程名
|
||||
%%
|
||||
%% @spec get_table_supervisor_name(Tab :: atom()) -> Worker :: atom().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_table_supervisor_name(Tab :: atom()) -> Worker :: atom().
|
||||
get_table_supervisor_name(Tab) ->
|
||||
list_to_atom("ms_cache_gs_" ++ atom_to_list(Tab) ++ "_sup").
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 缓存map数据转record
|
||||
%%
|
||||
%% @spec map_to_record(Map :: map() | tuple() | none) -> Record :: tuple() | none.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec map_to_record(Map :: map() | tuple() | none) -> Record :: tuple() | none.
|
||||
map_to_record(none) ->
|
||||
none;
|
||||
map_to_record(#{} = Map) ->
|
||||
Handler = ?TABLE_HANDLER(maps:get('_table', Map)),
|
||||
Handler:map_to_record(Map);
|
||||
map_to_record(Record) ->
|
||||
% Record.
|
||||
Tab = element(1, Record),
|
||||
Handler = ?TABLE_HANDLER(Tab),
|
||||
Handler:map_to_record(Record).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 缓存record数据转map
|
||||
%%
|
||||
%% @spec record_to_map(Record :: tuple() | map() | none) -> Map :: map() | none.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec record_to_map(Record :: tuple() | map() | none) -> Map :: map() | none.
|
||||
record_to_map(none) ->
|
||||
none;
|
||||
record_to_map(#{} = Map) ->
|
||||
% Map;
|
||||
Handler = ?TABLE_HANDLER(maps:get('_table', Map)),
|
||||
Handler:record_to_map(Map);
|
||||
record_to_map(Record) ->
|
||||
Tab = element(1, Record),
|
||||
Handler = ?TABLE_HANDLER(Tab),
|
||||
Handler:record_to_map(Record).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 转换数据格式为预定义格式
|
||||
%%
|
||||
%% @spec format_data(Data :: tuple() | map() | none) -> FmtData :: tuple() | map().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec format_data(Data :: tuple() | map() | none) -> FmtData :: tuple() | map() | none.
|
||||
format_data(none) ->
|
||||
none;
|
||||
format_data(Data) ->
|
||||
case ms_cache_config:get_env(data_fmt, record) of
|
||||
record ->
|
||||
map_to_record(Data);
|
||||
map ->
|
||||
record_to_map(Data)
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取当前时间
|
||||
%%
|
||||
%% @spec now_ms() -> NowSec :: integer().
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec now_ms() -> NowSec :: integer().
|
||||
now_ms() ->
|
||||
erlang:system_time(milli_seconds).
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 等待返回n次结果
|
||||
%%
|
||||
%% @spec wait_n_reply(SuccMsg :: any(), FailMsg :: any(), Count :: integer()) -> ok.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
wait_n_reply(_SuccMsg, _FailMsg, Count) when Count =< 0 ->
|
||||
ok;
|
||||
wait_n_reply(SuccMsg, FailMsg, Count) ->
|
||||
receive
|
||||
SuccMsg ->
|
||||
wait_n_reply(SuccMsg, FailMsg, Count-1);
|
||||
FailMsg ->
|
||||
throw(wait_n_reply_error)
|
||||
end.
|
||||
|
||||
wait_n_reply(_SuccMsg, _FailMsg, Count, _Timeout) when Count =< 0 ->
|
||||
ok;
|
||||
wait_n_reply(SuccMsg, FailMsg, Count, Timeout) ->
|
||||
receive
|
||||
SuccMsg ->
|
||||
wait_n_reply(SuccMsg, FailMsg, Count-1, Timeout);
|
||||
FailMsg ->
|
||||
throw(wait_n_reply_error)
|
||||
after Timeout ->
|
||||
throw(wait_n_reply_timeout)
|
||||
end.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取增加表字段的sql语句
|
||||
%%
|
||||
%% @spec get_add_column_sql(Tab :: atom(), ColumnInfo :: tuple()) -> {ok, SqlStr :: list()}.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_add_column_sql(Tab :: atom(), ColumnInfo :: tuple()) -> {ok, SqlStr :: list()}.
|
||||
get_add_column_sql(Table, ColumnInfo) ->
|
||||
ChangedSqlStr = column_changed_sql(ColumnInfo),
|
||||
SqlStr = io_lib:format("ALTER TABLE ~p ADD COLUMN ~s", [Table, ChangedSqlStr]),
|
||||
{?OK, SqlStr}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取删除表字段的sql语句
|
||||
%%
|
||||
%% @spec get_del_column_sql(Tab :: atom(), Column :: atom()) -> {ok, SqlStr :: list()}.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_del_column_sql(Tab :: atom(), Column :: atom()) -> {ok, SqlStr :: list()}.
|
||||
get_del_column_sql(Table, Column) ->
|
||||
SqlStr = io_lib:format("ALTER TABLE ~p drop COLUMN ~p", [Table, Column]),
|
||||
{?OK, SqlStr}.
|
||||
|
||||
%%--------------------------------------------------------------------
|
||||
%% @doc
|
||||
%% 获取更新表字段的sql语句
|
||||
%%
|
||||
%% @spec get_upd_column_sql(Tab :: atom(), OldColumn :: atom(), ColumnInfo :: tuple()) -> {ok, SqlStr :: list()}.
|
||||
%% @end
|
||||
%%--------------------------------------------------------------------
|
||||
-spec get_upd_column_sql(Tab :: atom(), OldColumn :: atom(), ColumnInfo :: tuple()) -> {ok, SqlStr :: list()}.
|
||||
get_upd_column_sql(Table, OldColumn, ColumnInfo) ->
|
||||
ChangedSqlStr = column_changed_sql(ColumnInfo),
|
||||
SqlStr = io_lib:format("ALTER TABLE ~p CHANGE COLUMN ~p ~s", [Table, OldColumn, ChangedSqlStr]),
|
||||
{?OK, SqlStr}.
|
||||
get_upd_column_sql(Table, ColumnInfo) ->
|
||||
OldColumn = erlang:element(1, ColumnInfo),
|
||||
{?OK, SqlStr} = get_upd_column_sql(Table, OldColumn, ColumnInfo),
|
||||
{?OK, SqlStr}.
|
||||
|
||||
column_changed_sql({Column, int, Len, Content, ?TRUE}) when Len >= 20 ->
|
||||
io_lib:format("~p bigint(20) NOT NULL DEFAULT '0' COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, int, Len, Content, ?TRUE}) when Len =< 4 ->
|
||||
io_lib:format("~p tinyint(4) NOT NULL DEFAULT '0' COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, int, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p int(11) NOT NULL DEFAULT '0' COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, binary, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p varchar(~p) BINARY NOT NULL DEFAULT '' COMMENT '~s'", [Column, Len, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, binary, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p blob NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, atom, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p varchar(~p) BINARY NOT NULL DEFAULT 'undefined' COMMENT '~s'", [Column, erlang:min(Len, 255), unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, list, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p varchar(~p) BINARY NOT NULL DEFAULT '[]' COMMENT '~s'", [Column, Len, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, list, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p text NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, tuple, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p varchar(~p) BINARY NOT NULL DEFAULT '{}' COMMENT '~s'", [Column, Len, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, tuple, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p text NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, map, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p varchar(~p) BINARY NOT NULL DEFAULT '#{}' COMMENT '~s'", [Column, Len, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, map, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p text NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, any, Len, Content, ?TRUE}) when Len > 0 ->
|
||||
io_lib:format("~p mediumblob NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, any, _Len, Content, ?TRUE}) ->
|
||||
io_lib:format("~p blob NOT NULL COMMENT '~s'", [Column, unicode:characters_to_binary(Content)]);
|
||||
column_changed_sql({Column, decimal, {M, S}, Content, ?TRUE}) ->
|
||||
io_lib:format("~p decimal(~p, ~p) NOT NULL DEFAULT '0' COMMENT '~s'", [Column, M, S, unicode:characters_to_binary(Content)]).
|
||||
@@ -0,0 +1,911 @@
|
||||
%%%%%-------------------------------------------------------------------
|
||||
%%%%% @copyright (C) 2019, <COMPANY>
|
||||
%%%%% @doc
|
||||
%%%%% cache组件模块内部方法实现
|
||||
%%%%% @end
|
||||
%%%%% Created : 2019-09-03 22:01:42
|
||||
%%%%%-------------------------------------------------------------------
|
||||
-module(ms_cache_test).
|
||||
-include("base.hrl").
|
||||
%%
|
||||
%%%% ====================================================================
|
||||
%%%% Include files
|
||||
%%%% ====================================================================
|
||||
%%-include("../../include/ms_cache.hrl").
|
||||
%%-include("../../../game_server/include/auto_gen/cache/role_c.hrl").
|
||||
%%-include("../../../game_server/include/auto_gen/cache/item_c.hrl").
|
||||
%%
|
||||
%%-compile(export_all).
|
||||
%%-compile(nowarn_export_all).
|
||||
%%
|
||||
%%-define(CACHE_LIST, []).
|
||||
%%-define(PREPARE_SELECT_LIST, []).
|
||||
%%-define(PREPARE_UPDATE_LIST, []).
|
||||
%%
|
||||
%%
|
||||
%%%% 测试环境
|
||||
%%%% Intel(R) Core(TM)i5-8500 CPU @ 3.00GHz 3.00GHz, 16G内存, win10操作系统
|
||||
%%%% 测试结果
|
||||
%%%% 单key表，role表
|
||||
%%%% 1、插入100000笔数据，1000个进程插入，每个进程插入100笔数据 ms_cache_analyse:spawn_test_insert(role, 1000, 100).
|
||||
%%%% prepare for init
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=141000, one_time = 1.41, sec_num = 709219, timeout = undefined
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=125000, one_time = 1.25, sec_num = 800000, timeout = undefined
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=140000, one_time = 1.40, sec_num = 714285, timeout = undefined
|
||||
%%%% prepare for init sync
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=328000, one_time = 3.28, sec_num = 304878, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=109999, one_time = 1.09999, sec_num = 909099, timeout = undefined
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=93999, one_time = 0.93999, sec_num = 1063841, timeout = undefined
|
||||
%%%% {role,insert}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%
|
||||
%%
|
||||
%%%% 2、查询100000次数据，三次测试,
|
||||
%%%% 查询测试 timer:tc(fun() -> ms_cache:init(role, any), [ms_cache:lookup(role, N) || N<-lists:seq(1,100000)], ok end).
|
||||
%%%% 有缓存并发1000进程，每个进程查100次不同数据（prepare for init）
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=172000, one_time = 1.72, sec_num = 581395, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=172000, one_time = 1.72, sec_num = 581395, timeout = undefined
|
||||
%%%% 有缓存并发1000进程，每个进程查100次不同数据（prepare for execute）
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=141000, one_time = 1.41, sec_num = 709219, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=172000, one_time = 1.72, sec_num = 581395, timeout = undefined
|
||||
%%
|
||||
%%%% 无缓存并发1000进程，每个进程查100次不同数据（prepare for init）
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=15063000, one_time = 150.63, sec_num = 6638, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=17187000, one_time = 171.87, sec_num = 5818, timeout = undefined
|
||||
%%%% 无缓存并发1000进程，每个进程查100次不同数据（prepare for execute）
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=16313000, one_time = 163.13, sec_num = 6130, timeout = undefined
|
||||
%%%% {role,lookup}, finish_num:100000, cost_time=17749921, one_time = 177.49921, sec_num = 5633, timeout = undefined
|
||||
%%
|
||||
%%%% 3、更新100000笔数据，1000个进程，每个进程更新100笔数据
|
||||
%%%% prepare for init
|
||||
%%%% {role,update}, finish_num:100000, cost_time=109000, one_time = 1.09, sec_num = 917431, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=141000, one_time = 1.41, sec_num = 709219, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=94000, one_time = 0.94, sec_num = 1063829, timeout = undefined
|
||||
%%%% prepare for init sync
|
||||
%%%% {role,update}, finish_num:100000, cost_time=375000, one_time = 3.75, sec_num = 266666, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=313000, one_time = 3.13, sec_num = 319488, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=328000, one_time = 3.28, sec_num = 304878, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {role,update}, finish_num:100000, cost_time=109000, one_time = 1.09, sec_num = 917431, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=141000, one_time = 1.41, sec_num = 709219, timeout = undefined
|
||||
%%%% {role,update}, finish_num:100000, cost_time=139999, one_time = 1.39999, sec_num = 714290, timeout = undefined
|
||||
%%
|
||||
%%%% 4、删除100000笔数据，启动1000个进程，每个进程删除100笔数据
|
||||
%%%% prepare for init
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=47000, one_time = 0.47, sec_num = 2127659, timeout = undefined
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=32000, one_time = 0.32, sec_num = 3125000, timeout = undefined
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=62000, one_time = 0.62, sec_num = 1612903, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=63000, one_time = 0.63, sec_num = 1587301, timeout = undefined
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=62999, one_time = 0.62999, sec_num = 1587326, timeout = undefined
|
||||
%%%% {role,delete}, finish_num:100000, cost_time=46000, one_time = 0.46, sec_num = 2173913, timeout = undefined
|
||||
%%
|
||||
%%%% 双key表，item表
|
||||
%%%% 1、插入100000笔数据，1000个进程插入，每个进程插入100笔数据（第一个key相同），
|
||||
%%%% prepare for init
|
||||
%%%% {item,insert}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {item,insert}, finish_num:100000, cost_time=125000, one_time = 1.25, sec_num = 800000, timeout = undefined
|
||||
%%%% {item,insert}, finish_num:100000, cost_time=109000, one_time = 1.09, sec_num = 917431, timeout = undefined
|
||||
%%%% {item,insert}, finish_num:100000, cost_time=94000, one_time = 0.94, sec_num = 1063829, timeout = undefined
|
||||
%%
|
||||
%%%% 2、查询100000次数据，三次测试
|
||||
%%%% 有缓存prepare for init
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=172000, one_time = 1.72, sec_num = 581395, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=156000, one_time = 1.56, sec_num = 641025, timeout = undefined
|
||||
%%%% 有缓存prepare for execute
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=172000, one_time = 1.72, sec_num = 581395, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=219000, one_time = 2.19, sec_num = 456621, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=157000, one_time = 1.57, sec_num = 636942, timeout = undefined
|
||||
%%
|
||||
%%%% 无缓存prepare for init（首次从数据库加载）
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=16422000, one_time = 164.22, sec_num = 6089, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=16656000, one_time = 166.56, sec_num = 6003, timeout = undefined
|
||||
%%%% 无缓存prepare for execute（首次从数据库加载）
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=16422000, one_time = 164.22, sec_num = 6089, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=16656000, one_time = 166.56, sec_num = 6003, timeout = undefined
|
||||
%%%% {item,lookup}, finish_num:100000, cost_time=16984000, one_time = 169.84, sec_num = 5887, timeout = undefined
|
||||
%%
|
||||
%%%% 3、更新100000笔数据，1000个进程更新，每个进程更新100笔数据
|
||||
%%%% prepare for init
|
||||
%%%% {item,update}, finish_num:100000, cost_time=110000, one_time = 1.1, sec_num = 909090, timeout = undefined
|
||||
%%%% {item,update}, finish_num:100000, cost_time=93000, one_time = 0.93, sec_num = 1075268, timeout = undefined
|
||||
%%%% {item,update}, finish_num:100000, cost_time=125000, one_time = 1.25, sec_num = 800000, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {item,update}, finish_num:100000, cost_time=109000, one_time = 1.09, sec_num = 917431, timeout = undefined
|
||||
%%%% {item,update}, finish_num:100000, cost_time=78000, one_time = 0.78, sec_num = 1282051, timeout = undefined
|
||||
%%%% {item,update}, finish_num:100000, cost_time=125000, one_time = 1.25, sec_num = 800000, timeout = undefined
|
||||
%%
|
||||
%%
|
||||
%%%% 4、删除100000笔数据，1000个进程删除，每个进程删除100笔数据
|
||||
%%%% prepare for init
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=47000, one_time = 0.47, sec_num = 2127659, timeout = undefined
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=47000, one_time = 0.47, sec_num = 2127659, timeout = undefined
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=62000, one_time = 0.62, sec_num = 1612903, timeout = undefined
|
||||
%%%% prepare for execute
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=47000, one_time = 0.47, sec_num = 2127659, timeout = undefined
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=62000, one_time = 0.62, sec_num = 1612903, timeout = undefined
|
||||
%%%% {item,delete}, finish_num:100000, cost_time=63000, one_time = 0.63, sec_num = 1587301, timeout = undefined
|
||||
%%
|
||||
%%start() ->
|
||||
%% application:set_env(ms_cache, db_driver, 'mysql-otp'),
|
||||
%% application:set_env(ms_cache, db_user, "uegame"),
|
||||
%% application:set_env(ms_cache, db_pass, "uegame"),
|
||||
%% application:set_env(ms_cache, db_host, "172.16.11.210"),
|
||||
%% application:set_env(ms_cache, db_port, 3306),
|
||||
%% application:set_env(ms_cache, db_name, "ms_cache_test"),
|
||||
%% application:set_env(ms_cache, db_encoding, utf8),
|
||||
%% application:set_env(ms_cache, db_select_pool_name, select_pool_name),
|
||||
%% application:set_env(ms_cache, db_select_pool_size, 10),
|
||||
%% application:set_env(ms_cache, db_update_pool_name, update_pool_name),
|
||||
%% application:set_env(ms_cache, db_update_pool_size, 10),
|
||||
%% ms_cache:start(?CACHE_LIST, ?PREPARE_SELECT_LIST, ?PREPARE_UPDATE_LIST),
|
||||
%% ok.
|
||||
%%
|
||||
%%handle_result(Table, Tag, Num, Cnt, NowTime) ->
|
||||
%% erlang:spawn(
|
||||
%% fun() ->
|
||||
%% erlang:put(opt_name, {Table, Tag}),
|
||||
%% erlang:put(finish_cnt, {Num, 0}),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% {CostTime, Timeout} = timer:tc(fun() -> loop_result() end),
|
||||
%% OneTime = erlang:max(0.001, CostTime / (Num*Cnt)),
|
||||
%% SecNum = erlang:floor(1000000 / OneTime),
|
||||
%% ?ERROR(?MODULE, "~p, finish_num:~p, cost_time=~p, one_time = ~p, sec_num = ~p, timeout = ~p",
|
||||
%% [{Table, Tag}, Num*Cnt, CostTime, OneTime, SecNum, Timeout])
|
||||
%% end).
|
||||
%%
|
||||
%%loop_result() ->
|
||||
%% receive {finish, _N, TimeoutCnt} ->
|
||||
%% add_timeout(TimeoutCnt),
|
||||
%% case add_finish() of
|
||||
%% {Max, Now} when Now >= Max ->
|
||||
%% erlang:get(add_timeout);
|
||||
%% {_Max, _Now} ->
|
||||
%% loop_result()
|
||||
%% end
|
||||
%% after 3000 ->
|
||||
%% erlang:get(add_timeout)
|
||||
%% end.
|
||||
%%
|
||||
%%add_finish() ->
|
||||
%% {Max, Now} = erlang:get(finish_cnt),
|
||||
%% erlang:put(finish_cnt, {Max, Now+1}),
|
||||
%% {Max, Now+1}.
|
||||
%%
|
||||
%%%% 插入测试
|
||||
%%%% item : 1W进程，每个进程10次插入，耗时516毫秒 |
|
||||
%%spawn_test_insert(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, insert, Num, Cnt, NowTime),
|
||||
%%
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_insert(Table, (N-1)*Cnt+1, N*Cnt)
|
||||
%% end
|
||||
%% )
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%loop_insert(_, Min, Max) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_insert(role, Min, Max) ->
|
||||
%% FnUId = erlang:list_to_binary("client_" ++ integer_to_list(Min) ),
|
||||
%% ms_cache:insert(
|
||||
%% role,
|
||||
%% #role{
|
||||
%% role_id=Min,
|
||||
%% fn_game_id = <<"1629775983825100">>,
|
||||
%% fn_platform = <<"0">>,
|
||||
%% fn_uid = FnUId,
|
||||
%% fn_name = FnUId,
|
||||
%% role_name = erlang:integer_to_binary(Min rem 100)
|
||||
%% }
|
||||
%% ),
|
||||
%% loop_insert(role, Min+1, Max);
|
||||
%%loop_insert(item, Min, Max) ->
|
||||
%% ms_cache:insert(item, #item{ role_id=Max, item_id=Min }),
|
||||
%% loop_insert(item, Min+1, Max).
|
||||
%%
|
||||
%%%% 查询测试
|
||||
%%%% 无缓存无数据 1W进程, 每个进程10次查询，三次耗时 21109 | | 毫秒
|
||||
%%%% 有缓存无数据 10W进程，100W次查询，耗时94秒 <==> 45W超时 | 1W进程，10W次查询，耗时7秒 <==> 0超时(10W进程10W查询差不多 5秒)
|
||||
%%%% 有缓存有数据 1进程，每个进程10次查询，三次耗时 236 | 219 | 218 毫秒
|
||||
%%
|
||||
%%spawn_test_lookup(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, lookup, Num, Cnt, NowTime),
|
||||
%%
|
||||
%%%% ms_tool:fprof_pid(whereis(ms_cache_gs_item_1), 20000),
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_lookup(Table, (N-1)*Cnt+1, N*Cnt)
|
||||
%% end
|
||||
%% )
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%loop_lookup(_, Min, Max) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%%%% Cnt = erlang:get(opt_cnt),
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_lookup(role, Min, Max) ->
|
||||
%% try
|
||||
%% ms_cache:lookup(role, Min)
|
||||
%% catch _E:M:T ->
|
||||
%% case M of
|
||||
%% {timeout, _} ->
|
||||
%% add_timeout();
|
||||
%% {badmatch, _} ->
|
||||
%% ok;
|
||||
%% _ ->
|
||||
%% ?ERROR(?MODULE, "M = ~p, Max = ~p, Trace = ~p", [M, Max, T])
|
||||
%% end
|
||||
%% end,
|
||||
%% loop_lookup(role, Min+1, Max);
|
||||
%%loop_lookup(item, Min, Max) ->
|
||||
%% try
|
||||
%% ms_cache:lookup(item, {Max, Min})
|
||||
%% catch _E:M:T ->
|
||||
%% case M of
|
||||
%% {timeout, _} ->
|
||||
%% add_timeout();
|
||||
%% {badmatch, _} ->
|
||||
%% ok;
|
||||
%% _ ->
|
||||
%% ?ERROR(?MODULE, "M = ~p, Max = ~p, Trace = ~p", [M, Max, T])
|
||||
%% end
|
||||
%% end,
|
||||
%% loop_lookup(item, Min+1, Max).
|
||||
%%
|
||||
%%%% 更新测试
|
||||
%%%% 10W进程，100W次更新，耗时 秒 <==> 0超时 | 1W进程，10W次更新，耗时1秒 <==> 10秒内查询超时
|
||||
%%spawn_test_update_element(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, update_element, Num, Cnt, NowTime),
|
||||
%%
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_update_element(Table, (N-1)*Cnt+1, N*Cnt, NowTime)
|
||||
%% end)
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%
|
||||
%%loop_update_element(_, Min, Max, _NowTime) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_update_element(role, Min, Max, NowTime) ->
|
||||
%% ms_cache:update_element(role, Min, {#role.exp, NowTime+Min}),
|
||||
%% loop_update_element(role, Min+1, Max, NowTime);
|
||||
%%loop_update_element(item, Min, Max, NowTime) ->
|
||||
%% ms_cache:update_element(item, {Max, Min}, {#item.number, NowTime+Min }),
|
||||
%% loop_update_element(item, Min+1, Max, NowTime).
|
||||
%%
|
||||
%%%% 更新测试
|
||||
%%%% 10W进程，100W次更新，耗时 秒 <==> 0超时 | 1W进程，10W次更新，耗时1秒 <==> 10秒内查询超时
|
||||
%%spawn_test_update(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, update, Num, Cnt, NowTime),
|
||||
%%
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_update(Table, (N-1)*Cnt+1, N*Cnt, NowTime)
|
||||
%% end)
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%
|
||||
%%loop_update(_, Min, Max, _NowTime) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_update(role, Min, Max, NowTime) ->
|
||||
%% ms_cache:update(role, #role{ role_id=Min,exp=NowTime+Min }),
|
||||
%% loop_update(role, Min+1, Max, NowTime);
|
||||
%%loop_update(item, Min, Max, NowTime) ->
|
||||
%% ms_cache:update(item, #item{ role_id=Max, item_id=Min,number=NowTime+Min }),
|
||||
%% loop_update(item, Min+1, Max, NowTime).
|
||||
%%
|
||||
%%%% 删除测试
|
||||
%%%% 10W进程，100W次删除，耗时4秒 <==> 0超时 | 1W进程，10W次删除，耗时0秒 <==> 10内超时
|
||||
%%spawn_test_delete(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, delete, Num, Cnt, NowTime),
|
||||
%%
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_delete(Table, (N-1)*Cnt+1, N*Cnt)
|
||||
%% end)
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%
|
||||
%%loop_delete(_, Min, Max) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_delete(role, Min, Max) ->
|
||||
%% ms_cache:delete(role, Min),
|
||||
%% loop_delete(role, Min+1, Max);
|
||||
%%loop_delete(item, Min, Max) ->
|
||||
%% ms_cache:delete(item, {Max, Min}),
|
||||
%% loop_delete(item, Min+1, Max).
|
||||
%%
|
||||
%%
|
||||
%%% test_lookup_by_key_index(Min, Max) ->
|
||||
%%% LookupCostTime = timer:tc(?MODULE, loop_lookup_by_key_index, [Min, Max, {0,0}]),
|
||||
%%% ?ERROR(?MODULE, "lookup_by_key_index_num:~p cost_time=~p", [Max-Min+1, LookupCostTime]).
|
||||
%%spawn_test_lookup_by_key_index(Table, Num, Cnt) ->
|
||||
%% NowTime = now_ms(),
|
||||
%% Pid = handle_result(Table, lookup_by_key_index, Num, Cnt, NowTime),
|
||||
%%
|
||||
%% F = fun(N) ->
|
||||
%% spawn(
|
||||
%% fun() ->
|
||||
%% ms_cache:init(Table, any),
|
||||
%% erlang:put(finish_pid, Pid),
|
||||
%% erlang:put(max_num, N*Cnt),
|
||||
%% erlang:put(start_time, NowTime),
|
||||
%% erlang:put(opt_cnt, Cnt),
|
||||
%% erlang:put(opt_seq, N),
|
||||
%% loop_lookup_by_key_index(Table, (N-1)*Cnt+1, N*Cnt)
|
||||
%% end)
|
||||
%% end,
|
||||
%% lists:foreach(F, lists:seq(1,Num)).
|
||||
%%
|
||||
%%loop_lookup_by_key_index(_, Min, Max) when Min =< 0;Max =< 0; Max<Min ->
|
||||
%% TimeoutCnt = erlang:get(add_timeout),
|
||||
%% ?IF(Max == erlang:get(max_num), erlang:get(finish_pid) ! {finish, erlang:get(opt_seq), TimeoutCnt});
|
||||
%%loop_lookup_by_key_index(item, Min, Max) ->
|
||||
%% try
|
||||
%% ms_cache:lookup_by_key_index(item, Max)
|
||||
%% catch _E:M:T ->
|
||||
%% case M of
|
||||
%% {timeout, _} ->
|
||||
%% add_timeout();
|
||||
%% {badmatch, _} ->
|
||||
%% ok;
|
||||
%% _ ->
|
||||
%% ?ERROR(?MODULE, "M = ~p, Max = ~p, Trace = ~p", [M, Max, T])
|
||||
%% end
|
||||
%% end,
|
||||
%% loop_lookup_by_key_index(item, Min+1, Max);
|
||||
%%loop_lookup_by_key_index(role, Min, Max) ->
|
||||
%% try
|
||||
%% ms_cache:lookup_by_key_index(role, Max)
|
||||
%% catch _E:M:T ->
|
||||
%% case M of
|
||||
%% {timeout, _} ->
|
||||
%% add_timeout();
|
||||
%% {badmatch, _} ->
|
||||
%% ok;
|
||||
%% _ ->
|
||||
%% ?ERROR(?MODULE, "M = ~p, Max = ~p, Trace = ~p", [M, Max, T])
|
||||
%% end
|
||||
%% end,
|
||||
%% loop_lookup_by_key_index(role, Min+1, Max).
|
||||
%%
|
||||
%%add_timeout() ->
|
||||
%% add_timeout(1).
|
||||
%%add_timeout(undefined) ->
|
||||
%% ok;
|
||||
%%add_timeout(Add) ->
|
||||
%% case erlang:get(add_timeout) of
|
||||
%% undefined ->
|
||||
%% erlang:put(add_timeout, Add);
|
||||
%% N ->
|
||||
%% erlang:put(add_timeout, N+Add)
|
||||
%% end.
|
||||
%%
|
||||
%%
|
||||
%%%% @doc 当前Unix时间戳(秒)
|
||||
%%now_sec() ->
|
||||
%% erlang:system_time(second).
|
||||
%%
|
||||
%%%% @doc 当前Unix时间戳(毫秒)
|
||||
%%now_ms() ->
|
||||
%% erlang:system_time(millisecond).
|
||||
%%
|
||||
%%
|
||||
%%
|
||||
%%%% ===================== 插入性能对比 =====================
|
||||
%%%%test_insert_ets , Num = 1000, CostTime = 512, QPS = 1953125
|
||||
%%%%test_insert_ets , Num = 1000, CostTime = 409, QPS = 2444987
|
||||
%%%%test_insert_ets , Num = 1000, CostTime = 819, QPS = 1221001
|
||||
%%
|
||||
%%%%test_insert_mnesia, Num = 1000, CostTime = 1536, QPS = 651041
|
||||
%%%%test_insert_mnesia, Num = 1000, CostTime = 1536, QPS = 651041
|
||||
%%%%test_insert_mnesia, Num = 1000, CostTime = 2560, QPS = 390625
|
||||
%%
|
||||
%%%%test_insert_dets , Num = 1000, CostTime = 56217, QPS = 17788
|
||||
%%%%test_insert_dets , Num = 1000, CostTime = 23859, QPS = 41912
|
||||
%%%%test_insert_dets , Num = 1000, CostTime = 24371, QPS = 41032
|
||||
%%
|
||||
%%%%test_insert_mysql , Num = 1000, CostTime = 2159922, QPS = 462
|
||||
%%%%test_insert_mysql , Num = 1000, CostTime = 2528050, QPS = 395
|
||||
%%%%test_insert_mysql , Num = 1000, CostTime = 2245733, QPS = 445
|
||||
%%%% ===================== 查询性能对比 =====================
|
||||
%%%%test_lookup_ets , Num = 1000, CostTime = 307, QPS = 3257328
|
||||
%%%%test_lookup_ets , Num = 1000, CostTime = 307, QPS = 3257328
|
||||
%%%%test_lookup_ets , Num = 1000, CostTime = 614, QPS = 1628664
|
||||
%%
|
||||
%%%%test_lookup_mnesia, Num = 1000, CostTime = 512, QPS = 1953125
|
||||
%%%%test_lookup_mnesia, Num = 1000, CostTime = 409, QPS = 2444987
|
||||
%%%%test_lookup_mnesia, Num = 1000, CostTime = 512, QPS = 1953125
|
||||
%%
|
||||
%%%%test_lookup_dets , Num = 1000, CostTime = 9420, QPS = 106157
|
||||
%%%%test_lookup_dets , Num = 1000, CostTime = 8908, QPS = 112258
|
||||
%%%%test_lookup_dets , Num = 1000, CostTime = 8499, QPS = 117660
|
||||
%%
|
||||
%%%%test_lookup_mysql , Num = 1000, CostTime = 817356, QPS = 1223
|
||||
%%%%test_lookup_mysql , Num = 1000, CostTime = 834764, QPS = 1197
|
||||
%%%%test_lookup_mysql , Num = 1000, CostTime = 888319, QPS = 1125
|
||||
%%%% ===================== 更新性能对比 =====================
|
||||
%%%%test_update_ets , Num = 1000, CostTime = 307, QPS = 3257328
|
||||
%%%%test_update_ets , Num = 1000, CostTime = 307, QPS = 3257328
|
||||
%%%%test_update_ets , Num = 1000, CostTime = 512, QPS = 1953125
|
||||
%%
|
||||
%%%%test_update_mnesia, Num = 1000, CostTime = 1331, QPS = 751314
|
||||
%%%%test_update_mnesia, Num = 1000, CostTime = 1433, QPS = 697836
|
||||
%%%%test_update_mnesia, Num = 1000, CostTime = 1536, QPS = 651041
|
||||
%%
|
||||
%%%%test_update_dets , Num = 1000, CostTime = 11571, QPS = 86422
|
||||
%%%%test_update_dets , Num = 1000, CostTime = 11878, QPS = 84189
|
||||
%%%%test_update_dets , Num = 1000, CostTime = 11264, QPS = 88778
|
||||
%%
|
||||
%%%%test_update_mysql , Num = 1000, CostTime = 2192383, QPS = 456
|
||||
%%%%test_update_mysql , Num = 1000, CostTime = 2403429, QPS = 416
|
||||
%%%%test_update_mysql , Num = 1000, CostTime = 2677656, QPS = 373
|
||||
%%%% ===================== 删除性能对比 =====================
|
||||
%%%%test_delete_ets , Num = 1000, CostTime = 102, QPS = 3257328
|
||||
%%%%test_delete_ets , Num = 1000, CostTime = 102, QPS = 3257328
|
||||
%%%%test_delete_ets , Num = 1000, CostTime = 204, QPS = 1628664
|
||||
%%
|
||||
%%%%test_delete_mnesia, Num = 1000, CostTime = 819, QPS = 1953125
|
||||
%%%%test_delete_mnesia, Num = 1000, CostTime = 1024, QPS = 2444987
|
||||
%%%%test_delete_mnesia, Num = 1000, CostTime = 921, QPS = 1953125
|
||||
%%
|
||||
%%%%test_delete_dets , Num = 1000, CostTime = 14950, QPS = 106157
|
||||
%%%%test_delete_dets , Num = 1000, CostTime = 15052, QPS = 112258
|
||||
%%%%test_delete_dets , Num = 1000, CostTime = 14643, QPS = 117660
|
||||
%%
|
||||
%%%%test_delete_mysql , Num = 1000, CostTime = 1655295, QPS = 1223
|
||||
%%%%test_delete_mysql , Num = 1000, CostTime = 2399231, QPS = 1197
|
||||
%%%%test_delete_mysql , Num = 1000, CostTime = 1866956, QPS = 1125
|
||||
%%
|
||||
%%test_init() ->
|
||||
%% filelib:ensure_dir("dets/*"),
|
||||
%% {?OK, Options} = ms_cache_gs:merge_options([], role_c),
|
||||
%% ms_cache_edb:create_table(e_test_role, Options),
|
||||
%% ms_cache_edb2:create_table(m_test_role, [{type, set}, {attributes, role_c:field_list()}, {record_name, role}, {ram_copies, [erlang:node()]}]),
|
||||
%% ms_cache_ddb:create_table(d_test_role, []),
|
||||
%% ok.
|
||||
%%
|
||||
%%test_insert(Num) ->
|
||||
%% lists:foreach(
|
||||
%% fun({Mod, Table}) ->
|
||||
%% NowTime11 = now_sec(),
|
||||
%% {Time11, ?OK} = timer:tc(fun() -> test_insert_all(NowTime11, Num, Mod, Table) end),
|
||||
%% NowTime12 = now_sec(),
|
||||
%% {Time12, ?OK} = timer:tc(fun() -> test_update_all(NowTime12, Num, Mod, Table) end),
|
||||
%% {Time13, ?OK} = timer:tc(fun() -> test_lookup_all(Num, Mod, Table) end),
|
||||
%% {Time14, ?OK} = timer:tc(fun() -> test_delete_all(Num, Mod, Table) end),
|
||||
%%
|
||||
%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%% ?ERROR(?MODULE, "~p insert , Num = ~p, CostTime = ~p, QPS = ~p", [Table, Num, Time11, erlang:trunc(1000000/(Time11/Num))]),
|
||||
%% ?ERROR(?MODULE, "~p lookup , Num = ~p, CostTime = ~p, QPS = ~p", [Table, Num, Time13, erlang:trunc(1000000/(Time13/Num))]),
|
||||
%% ?ERROR(?MODULE, "~p update , Num = ~p, CostTime = ~p, QPS = ~p", [Table, Num, Time12, erlang:trunc(1000000/(Time12/Num))]),
|
||||
%% ?ERROR(?MODULE, "~p delete , Num = ~p, CostTime = ~p, QPS = ~p", [Table, Num, Time14, erlang:trunc(1000000/(Time13/Num))]),
|
||||
%%
|
||||
%% erlang:garbage_collect()
|
||||
%% end, [{ets, e_test_role}, {ms_cache_mdb, m_test_role}, {ms_cache_ddb, d_test_role}]
|
||||
%% ),
|
||||
%%
|
||||
%% NowTime31 = now_sec(),
|
||||
%% {Time31, ?OK} = timer:tc(fun() -> test_insert_mysql(NowTime31, Num) end),
|
||||
%% NowTime32 = now_sec(),
|
||||
%% {Time32, ?OK} = timer:tc(fun() -> test_update_mysql(NowTime32, Num) end),
|
||||
%% {Time33, ?OK} = timer:tc(fun() -> test_lookup_mysql(Num) end),
|
||||
%% {Time34, ?OK} = timer:tc(fun() -> test_delete_mysql(Num) end),
|
||||
%%
|
||||
%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%% ?ERROR(?MODULE, "test_insert_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time31, erlang:trunc(1000000/(Time31/Num))]),
|
||||
%% ?ERROR(?MODULE, "test_lookup_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time33, erlang:trunc(1000000/(Time33/Num))]),
|
||||
%% ?ERROR(?MODULE, "test_update_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time32, erlang:trunc(1000000/(Time32/Num))]),
|
||||
%% ?ERROR(?MODULE, "test_delete_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time34, erlang:trunc(1000000/(Time33/Num))]),
|
||||
%%
|
||||
%%
|
||||
%%%% NowTime11 = now_sec(),
|
||||
%%%% {Time11, ?OK} = timer:tc(fun() -> test_insert_ets(NowTime11, Num) end),
|
||||
%%%% NowTime12 = now_sec(),
|
||||
%%%% {Time12, ?OK} = timer:tc(fun() -> test_update_ets(NowTime12, Num) end),
|
||||
%%%% {Time13, ?OK} = timer:tc(fun() -> test_lookup_ets(Num) end),
|
||||
%%%% {Time14, ?OK} = timer:tc(fun() -> test_delete_ets(Num) end),
|
||||
%%%% erlang:garbage_collect(),
|
||||
%%%%
|
||||
%%%% NowTime41 = now_sec(),
|
||||
%%%% {Time41, ?OK} = timer:tc(fun() -> test_insert_mnesia(NowTime41, Num) end),
|
||||
%%%% NowTime42 = now_sec(),
|
||||
%%%% {Time42, ?OK} = timer:tc(fun() -> test_update_mnesia(NowTime42, Num) end),
|
||||
%%%% {Time43, ?OK} = timer:tc(fun() -> test_lookup_mnesia(Num) end),
|
||||
%%%% {Time44, ?OK} = timer:tc(fun() -> test_delete_mnesia(Num) end),
|
||||
%%%% erlang:garbage_collect(),
|
||||
%%%%
|
||||
%%%% NowTime21 = now_sec(),
|
||||
%%%% {Time21, ?OK} = timer:tc(fun() -> test_insert_dets(NowTime21, Num) end),
|
||||
%%%% NowTime22 = now_sec(),
|
||||
%%%% {Time22, ?OK} = timer:tc(fun() -> test_update_dets(NowTime22, Num) end),
|
||||
%%%% {Time23, ?OK} = timer:tc(fun() -> test_lookup_dets(Num) end),
|
||||
%%%% {Time24, ?OK} = timer:tc(fun() -> test_delete_dets(Num) end),
|
||||
%%%% erlang:garbage_collect(),
|
||||
%%
|
||||
%%%% NowTime31 = now_sec(),
|
||||
%%%% {Time31, ?OK} = timer:tc(fun() -> test_insert_mysql(NowTime31, Num) end),
|
||||
%%%% NowTime32 = now_sec(),
|
||||
%%%% {Time32, ?OK} = timer:tc(fun() -> test_update_mysql(NowTime32, Num) end),
|
||||
%%%% {Time33, ?OK} = timer:tc(fun() -> test_lookup_mysql(Num) end),
|
||||
%%%% {Time34, ?OK} = timer:tc(fun() -> test_delete_mysql(Num) end),
|
||||
%%%% erlang:garbage_collect(),
|
||||
%%%%
|
||||
%%%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%%%% ?ERROR(?MODULE, "test_insert_ets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time11, erlang:trunc(1000000/(Time11/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_insert_mnesia, Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time41, erlang:trunc(1000000/(Time41/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_insert_dets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time21, erlang:trunc(1000000/(Time21/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_insert_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time31, erlang:trunc(1000000/(Time31/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%%%% ?ERROR(?MODULE, "test_lookup_ets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time13, erlang:trunc(1000000/(Time13/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_lookup_mnesia, Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time43, erlang:trunc(1000000/(Time43/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_lookup_dets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time23, erlang:trunc(1000000/(Time23/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_lookup_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time33, erlang:trunc(1000000/(Time33/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%%%% ?ERROR(?MODULE, "test_update_ets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time12, erlang:trunc(1000000/(Time12/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_update_mnesia, Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time42, erlang:trunc(1000000/(Time42/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_update_dets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time22, erlang:trunc(1000000/(Time22/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_update_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time32, erlang:trunc(1000000/(Time32/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "==========================================================================================================", []),
|
||||
%%%% ?ERROR(?MODULE, "test_delete_ets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time14, erlang:trunc(1000000/(Time13/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_delete_mnesia, Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time44, erlang:trunc(1000000/(Time43/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_delete_dets , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time24, erlang:trunc(1000000/(Time23/Num))]),
|
||||
%%%% ?ERROR(?MODULE, "test_delete_mysql , Num = ~p, CostTime = ~p, QPS = ~p", [Num, Time34, erlang:trunc(1000000/(Time33/Num))]),
|
||||
%%
|
||||
%% ets:delete_all_objects(e_test_role),
|
||||
%% ms_cache_edb2:delete_all_objects(m_test_role),
|
||||
%% ms_cache_ddb:delete_all_objects(d_test_role),
|
||||
%% role_c:delete_all(?UPDATE_POOL),
|
||||
%%
|
||||
%% erlang:garbage_collect(),
|
||||
%%
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_insert_all(_NowSec, Num, _Mod, _Table) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_insert_all(NowSec, Num, Mod, Table) ->
|
||||
%% Mod:insert(Table, new_role(Num, NowSec)),
|
||||
%% test_insert_all(NowSec, Num-1, Mod, Table).
|
||||
%%
|
||||
%%test_update_all(_NowSec, Num, _Mod, _Table) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_update_all(NowSec, Num, Mod, Table) ->
|
||||
%% Mod:insert(Table, new_role(Num, NowSec)),
|
||||
%% test_update_all(NowSec, Num-1, Mod, Table).
|
||||
%%
|
||||
%%test_lookup_all(Num, _Mod, _Table) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_lookup_all(Num, Mod, Table) ->
|
||||
%% Mod:lookup(Table, Num),
|
||||
%% test_lookup_all(Num-1, Mod, Table),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_delete_all(Num, _Mod, _Table) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_delete_all(Num, Mod, Table) ->
|
||||
%% Mod:delete(Table, Num),
|
||||
%% test_delete_all(Num-1, Mod, Table),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_insert_ets(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_insert_ets(NowSec, Num) ->
|
||||
%% ets:insert(
|
||||
%% e_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_insert_ets(NowSec, Num-1).
|
||||
%%
|
||||
%%test_update_ets(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_update_ets(NowSec, Num) ->
|
||||
%% ets:insert(
|
||||
%% e_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_update_ets(NowSec, Num-1).
|
||||
%%
|
||||
%%test_lookup_ets(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_lookup_ets(Num) ->
|
||||
%% ets:lookup(e_test_role, Num),
|
||||
%% test_lookup_ets(Num-1),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_delete_ets(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_delete_ets(Num) ->
|
||||
%% ets:delete(e_test_role, Num),
|
||||
%% test_delete_ets(Num-1),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%
|
||||
%%test_insert_mnesia(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_insert_mnesia(NowSec, Num) ->
|
||||
%% ms_cache_edb2:insert(
|
||||
%% m_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_insert_mnesia(NowSec, Num-1).
|
||||
%%
|
||||
%%test_update_mnesia(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_update_mnesia(NowSec, Num) ->
|
||||
%% ms_cache_edb2:insert(
|
||||
%% m_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_insert_mnesia(NowSec, Num-1).
|
||||
%%
|
||||
%%test_lookup_mnesia(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_lookup_mnesia(Num) ->
|
||||
%% ms_cache_edb2:lookup(m_test_role, Num),
|
||||
%% test_lookup_mnesia(Num-1),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_delete_mnesia(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_delete_mnesia(Num) ->
|
||||
%% ms_cache_edb2:delete(m_test_role, Num),
|
||||
%% test_delete_mnesia(Num-1),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_insert_dets(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_insert_dets(NowSec, Num) ->
|
||||
%% ms_cache_ddb:insert(
|
||||
%% d_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_insert_dets(NowSec, Num-1).
|
||||
%%
|
||||
%%test_update_dets(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_update_dets(NowSec, Num) ->
|
||||
%% ms_cache_ddb:insert(
|
||||
%% d_test_role,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_update_dets(NowSec, Num-1).
|
||||
%%
|
||||
%%test_lookup_dets(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_lookup_dets(Num) ->
|
||||
%% ms_cache_ddb:lookup(d_test_role, Num),
|
||||
%% test_lookup_dets(Num - 1).
|
||||
%%
|
||||
%%test_delete_dets(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_delete_dets(Num) ->
|
||||
%% ms_cache_ddb:delete(d_test_role, Num),
|
||||
%% test_delete_dets(Num - 1).
|
||||
%%
|
||||
%%test_insert_mysql(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_insert_mysql(NowSec, Num) ->
|
||||
%% role_c:insert(
|
||||
%% ?UPDATE_POOL,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_insert_mysql(NowSec, Num-1).
|
||||
%%
|
||||
%%test_update_mysql(_NowSec, Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_update_mysql(NowSec, Num) ->
|
||||
%% role_c:update(
|
||||
%% ?UPDATE_POOL,
|
||||
%% new_role(Num, NowSec)
|
||||
%% ),
|
||||
%% test_update_mysql(NowSec, Num-1).
|
||||
%%
|
||||
%%test_lookup_mysql(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_lookup_mysql(Num) ->
|
||||
%% role_c:select_by_key(?SELECT_POOL, Num),
|
||||
%% test_lookup_mysql(Num - 1).
|
||||
%%
|
||||
%%test_delete_mysql(Num) when Num =< 0 ->
|
||||
%% ?OK;
|
||||
%%test_delete_mysql(Num) ->
|
||||
%% role_c:delete(?UPDATE_POOL, Num),
|
||||
%% test_delete_mysql(Num - 1).
|
||||
%%
|
||||
%%new_role(Num, NowSec) ->
|
||||
%% #role{
|
||||
%% '_id' = Num,
|
||||
%% role_id=Num,
|
||||
%% fn_game_id = <<"1629775983825100">>,
|
||||
%% fn_platform = <<"0">>,
|
||||
%% fn_uid = <<"test">>,
|
||||
%% fn_name = <<"test">>,
|
||||
%% role_name = erlang:integer_to_binary(Num rem 255),
|
||||
%% last_login_time = NowSec,
|
||||
%% last_logout_time = NowSec
|
||||
%% }.
|
||||
%%
|
||||
%%test_ets_index(Max, IndexNum) ->
|
||||
%% %% 删除测试的内存表
|
||||
%% catch ms_cache_edb2:close(ets_test_bag),
|
||||
%% catch ms_cache_edb3:close(ets_test_set),
|
||||
%% catch ms_cache_mdb:close(mnesia_test),
|
||||
%% catch ms_cache_edb:close(ets_test_ordered_set),
|
||||
%%
|
||||
%% %% 创建测试的内存表
|
||||
%% {?OK, Options} = ms_cache_gs:merge_options([], role_c),
|
||||
%% ms_cache_edb2:create_table(ets_test_bag, Options),
|
||||
%% ms_cache_edb3:create_table(ets_test_set, Options),
|
||||
%% ms_cache_mdb:create_table(mnesia_test, Options),
|
||||
%% ms_cache_edb:create_table(ets_test_ordered_set, Options),
|
||||
%% %% 查询玩家数据
|
||||
%% Now = now_sec(),
|
||||
%% Roles = [new_role(RoleId, Now) || RoleId <- lists:seq(1, Max)],
|
||||
%%
|
||||
%% %% insert
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:insert_new(ets_test_bag, Role) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:insert_new(ets_test_set, Role) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:insert_new(mnesia_test, Role) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:insert_new(ets_test_ordered_set, Role) end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag insert Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag, TimeSet, TimeMnesia, TimeEtsOrder]),
|
||||
%%
|
||||
%% %% update
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag0, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:insert(ets_test_bag, Role#role{ last_login_time = Role#role.last_login_time+1 }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet0, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:insert(ets_test_set, Role#role{ last_login_time = Role#role.last_login_time+1 }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia0, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:insert(mnesia_test, Role#role{ last_login_time = Role#role.last_login_time+1 }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder0, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:insert(ets_test_ordered_set, Role#role{ last_login_time = Role#role.last_login_time+1 }) end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag update Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag0, TimeSet0, TimeMnesia0, TimeEtsOrder0]),
|
||||
%%
|
||||
%% %% update index
|
||||
%% Rem = ?IF(IndexNum > Max, Max, Max div IndexNum),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag4, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:insert(ets_test_bag, Role#role{ role_name = erlang:integer_to_binary(Role#role.role_id rem Rem) }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet4, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:insert(ets_test_set, Role#role{ role_name = erlang:integer_to_binary(Role#role.role_id rem Rem) }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia4, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:insert(mnesia_test, Role#role{ role_name = erlang:integer_to_binary(Role#role.role_id rem Rem) }) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder4, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:insert(ets_test_ordered_set, Role#role{ role_name = erlang:integer_to_binary(Role#role.role_id rem Rem) }) end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag update_index Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag4, TimeSet4, TimeMnesia4, TimeEtsOrder4]),
|
||||
%%
|
||||
%% %% lookup
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag2, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:lookup(ets_test_bag, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet2, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:lookup(ets_test_set, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia2, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:lookup(mnesia_test, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder2, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:lookup(ets_test_ordered_set, Role#role.'_id') end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag lookup Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag2, TimeSet2, TimeMnesia2, TimeEtsOrder2]),
|
||||
%%
|
||||
%% %% lookup_by_key_index
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag3, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:index_read(ets_test_bag, Role#role.role_id, #role.role_id) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet3, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:index_read(ets_test_set, Role#role.role_id, #role.role_id) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia3, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:index_read(mnesia_test, Role#role.role_id, #role.role_id) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder3, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:index_read(ets_test_ordered_set, Role#role.role_id, #role.role_id) end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag lookup_by_key_index Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag3, TimeSet3, TimeMnesia3, TimeEtsOrder3]),
|
||||
%%
|
||||
%% %% index_read
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag5, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:index_read(ets_test_bag, Role#role.role_name, #role.role_name) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet5, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:index_read(ets_test_set, Role#role.role_name, #role.role_name) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia5, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:index_read(mnesia_test, Role#role.role_name, #role.role_name) end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder5, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:index_read(ets_test_ordered_set, Role#role.role_name, #role.role_name) end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag index_read Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag5, TimeSet5, TimeMnesia5, TimeEtsOrder5]),
|
||||
%%
|
||||
%% %% delete
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeBag1, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb2:delete(ets_test_bag, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeSet1, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb3:delete(ets_test_set, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeMnesia1, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_mdb:delete(mnesia_test, Role#role.'_id') end, Roles]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {TimeEtsOrder1, _} = timer:tc(lists, foreach, [fun(Role) -> ms_cache_edb:delete(ets_test_ordered_set, Role#role.'_id') end, Roles]),
|
||||
%% ?INFO(?MODULE, "ets_test_bag delete Len = ~p, TimeBag = ~p, TimeSet = ~p, TimeMnesia = ~p, TimeEtsOrder = ~p",
|
||||
%% [erlang:length(Roles), TimeBag1, TimeSet1, TimeMnesia1, TimeEtsOrder1]),
|
||||
%%
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_role_batch() ->
|
||||
%% Roles = role_c:select_all(select_pool_name),
|
||||
%% test_role_batch(Roles, 1),
|
||||
%% test_role_batch(Roles, 1*1024),
|
||||
%% test_role_batch(Roles, 8*1024),
|
||||
%% test_role_batch(Roles, 16*1024),
|
||||
%% test_role_batch(Roles, 32*1024),
|
||||
%% test_role_batch(Roles, 64*1024),
|
||||
%% test_role_batch(Roles, 128*1024),
|
||||
%% test_role_batch(Roles, 256*1024),
|
||||
%% test_role_batch(Roles, 512*1024),
|
||||
%% test_role_batch(Roles, 1024*1024),
|
||||
%% test_role_batch(Roles, 2048*1024),
|
||||
%% test_role_batch(Roles, 4096*1024),
|
||||
%% ?OK.
|
||||
%%
|
||||
%%test_role_batch(Roles, BatchLen) ->
|
||||
%% RolesLen = erlang:length(Roles),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {Time1, _} = timer:tc(role_c, batch_insert, [update_pool_name, Roles, BatchLen]),
|
||||
%% ?INFO(?MODULE, "test_role_batch BatchLen = ~p, Time = ~p, RolesLen = ~p, QPS = ~p",
|
||||
%% [BatchLen div 1024, Time1, RolesLen, erlang:trunc(100000/(Time1/RolesLen))]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {Time2, _} = timer:tc(role_c, batch_insert, [update_pool_name, Roles, BatchLen]),
|
||||
%% ?INFO(?MODULE, "test_role_batch BatchLen = ~p, Time = ~p, RolesLen = ~p, QPS = ~p",
|
||||
%% [BatchLen div 1024, Time2, RolesLen, erlang:trunc(100000/(Time2/RolesLen))]),
|
||||
%% erlang:garbage_collect(erlang:self()),
|
||||
%% {Time3, _} = timer:tc(role_c, batch_insert, [update_pool_name, Roles, BatchLen]),
|
||||
%% ?INFO(?MODULE, "test_role_batch BatchLen = ~p, Time = ~p, RolesLen = ~p, QPS = ~p",
|
||||
%% [BatchLen div 1024, Time3, RolesLen, erlang:trunc(100000/(Time3/RolesLen))]),
|
||||
%% ?OK.
|
||||
Reference in New Issue
Block a user