diff --git a/kaggle_run.py b/kaggle_run.py index c70abd9..b8d6081 100644 --- a/kaggle_run.py +++ b/kaggle_run.py @@ -53,8 +53,9 @@ def get_ka(): return ka -def pull_push(): +def get_kernels(): """ + 获取配置信息、用户、内核 https://github.com/Kaggle/kaggle-api#kernels 其中cli后缀函数是用于shell中执行： kernel_pull(self, user_name, kernel_slug, kwargs) @@ -75,10 +76,36 @@ def pull_push(): kernels_status_cli(self, kernel, kernel_opt) """ ka = get_ka() - user = ka.config_values[ka.CONFIG_NAME_USER] # 获取当前用户 - kls = ka.kernels_list(user=user) # 指定用户的内核 + return ka, kls, user + + +def download_kernels(): + """ + 下载所有内核 + :return: + """ + ka, kls, user = get_kernels() + print(kls) + # kls_list = [kls[i:i + 6] for i in range(0, len(kls), 6)] + for kl in kls: + name = re.sub(r"\.|_", "-", str(kl)) # 只能包含字母数字和- + fully_name = user + "/" + name + try: + res = ka.kernels_status(fully_name) # 获取内核运行状态 + print(res) + ka.kernels_pull(fully_name, "kaggle_kernels") + except ApiException as e: + print(fully_name, e.status, e.reason) + + +def pull_push(): + """ + 下载元数据并推送 + :return: + """ + ka, kls, user = get_kernels() print(kls) # kls_list = [kls[i:i + 6] for i in range(0, len(kls), 6)] running_qty = 0 @@ -198,10 +225,7 @@ def kernels_initialize(api, name, folder): if __name__ == '__main__': - # run() - # 第一个参数是时间间隔（单位是秒，只有秒），第二个参数是要调用的函数名，第三个参数是调用函数的参数(tuple) - # t = Timer(inc, run, (inc,)) - # t.start() + # download_kernels() # 注意 sched 模块不是循环的，一次调度被执行后就结束了，如果想再执行，请再次 enter # 四个参数分别为：间隔事件、优先级（用于同时间到达的两个事件同时执行时定序）、被调用触发的函数，给该触发函数的参数（tuple形式） schedule.enter(0, 0, pull_push) # 提交拉取的内核 diff --git a/ping_oracle.py b/ping_oracle.py new file mode 100644 index 0000000..ec7cb1c --- /dev/null +++ b/ping_oracle.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# @Author : https://www.bajins.com +# @Description : 获取Oracle云各个区域的延迟 +# @File : ping_oracle.py +# @Version: 1.1.0 +# @Time : 2020/7/26 11:00 +# @Project: scripts_python +# @Package: +# @Software: PyCharm +from collections import namedtuple +from pythonping import ping +from prettytable import PrettyTable + +lists = [ + {"region": "亚太地区", "list": [ + {"area": "日本东部 东京", "addr": "objectstorage.ap-tokyo-1.oraclecloud.com"}, + {"area": "日本中部 大阪", "addr": "objectstorage.ap-osaka-1.oraclecloud.com"}, + {"area": "韩国中部 首尔", "addr": "objectstorage.ap-seoul-1.oraclecloud.com"}, + {"area": "韩国北部 春川", "addr": "objectstorage.ap-chuncheon-1.oraclecloud.com"}, + {"area": "澳大利亚东部 悉尼", "addr": "objectstorage.ap-sydney-1.oraclecloud.com"}, + {"area": "澳大利亚东南部 墨尔本", "addr": "objectstorage.ap-melbourne-1.oraclecloud.com"}, + {"area": "印度西部 孟买", "addr": "objectstorage.ap-mumbai-1.oraclecloud.com"}, + {"area": "印度南部 海得拉巴", "addr": "objectstorage.ap-hyderabad-1.oraclecloud.com"} + ] + }, + {"region": "北美地区", "list": [ + {"area": "美国东部 阿什本", "addr": "objectstorage.us-ashburn-1.oraclecloud.com"}, + {"area": "美国西部 凤凰城", "addr": "objectstorage.us-phoenix-1.oraclecloud.com"}, + {"area": "美国西部 圣何塞", "addr": "objectstorage.us-sanjose-1.oraclecloud.com"}, + {"area": "加拿大东南部 蒙特利尔", "addr": "objectstorage.ca-montreal-1.oraclecloud.com"}, + {"area": "加拿大东南部 多伦多", "addr": "objectstorage.ca-toronto-1.oraclecloud.com"} + ] + }, + {"region": "欧洲地区", "list": [ + {"area": "英国南部 伦敦", "addr": "objectstorage.uk-london-1.oraclecloud.com"}, + {"area": "英国西部 加的夫", "addr": "objectstorage.uk-cardiff-1.oraclecloud.com"}, + {"area": "德国中部 法兰克福", "addr": "objectstorage.eu-frankfurt-1.oraclecloud.com"}, + {"area": "瑞士北部 苏黎世", "addr": "objectstorage.eu-zurich-1.oraclecloud.com"}, + {"area": "荷兰西北部 阿姆斯特丹", "addr": "objectstorage.eu-amsterdam-1.oraclecloud.com"} + ] + }, + {"region": "中东地区", "list": [ + {"area": "阿联酋东部 迪拜", "addr": "objectstorage.me-dubai-1.oraclecloud.com"}, + {"area": "沙特阿拉伯西部 吉达", "addr": "objectstorage.me-jeddah-1.oraclecloud.com"} + ] + }, + {"region": "南美地区", "list": [ + {"area": "巴西东部 圣保罗", "addr": "objectstorage.sa-saopaulo-1.oraclecloud.com"}, + {"area": "智利中部 圣地亚哥", "addr": "objectstorage.sa-santiago-1.oraclecloud.com"} + ] + } +] + + +def ping_check(addr): + from tcping import Ping + ping = Ping(addr) + ping.ping(10) + + ret = ping.result.rows + for r in ret: + print(r) + + ret = ping.result.raw + print(ret) + + ret = ping.result.table + print(ret) + + +x = PrettyTable() + +Statistics = namedtuple('Statistics', ["地域", "地区", "Host", "Minimum", "Average", "Maximum"]) +x.field_names = Statistics._fields + +for ls in lists: + region = ls.get("region") + for ds in ls.get("list"): + area = ds.get("area") + addr = ds.get("addr") + try: + res = ping(addr, count=10, verbose=False, out=None) + x.add_row(Statistics(region, area, addr, res.rtt_min_ms, res.rtt_avg_ms, res.rtt_max_ms)) + except: + x.add_row(Statistics(region, area, addr, 0, 0, 0)) + + # ping_check(addr) + +# x.align = "l" +x.padding_width = 5 +print(x.get_string(sortby="Average", reversesort=False)) diff --git a/rclone_expect_config.py b/rclone_expect_config.py index afbd104..33ac28c 100644 --- a/rclone_expect_config.py +++ b/rclone_expect_config.py @@ -30,14 +30,18 @@ def run_cmd(cmd, popen=True, daemon=False, log_path="rclone.log"): 执行命令并根据参数决定是否在控制台输出执行结果 :param cmd: 执行的命令 :param popen: 是否回显，默认显示回显到控制台 - :param daemon: 是否守护进程，默认不启用，当popen=True时才启用 - :param log_path: 日志文件路径，在popen=False时启用 + :param daemon: 是否守护进程（后台运行），默认不启用 + :param log_path: 日志文件路径，在popen=False且daemon=True时启用 :return: """ - if not popen: # 执行命令不输出回显并保存执行结果到日志文件（后台运行） - call = subprocess.call(f'nohup {cmd} >{log_path} &', shell=True) - if call != 0: - print(f"执行失败，请查看{log_path}中的日志") + if not popen: + if daemon: + # 执行命令不输出回显并保存执行结果到日志文件（后台运行） + call = subprocess.call(f'nohup {cmd} >{log_path} &', shell=True) + if call != 0: + print(f"执行失败，请查看{log_path}中的日志") + else: + print(subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)) else: # 执行shell命令并实时输出回显 if daemon: # 将当前进程fork为一个守护进程 @@ -45,9 +49,8 @@ def run_cmd(cmd, popen=True, daemon=False, log_path="rclone.log"): if pid > 0: # 父进程退出 sys.exit(0) - # universal_newlines=True, bufsize=1 - process = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + # universal_newlines=True, bufsize=1, stdin=subprocess.PIPE, stdout=subprocess.PIPE + process = subprocess.Popen(cmd, shell=True, stderr=subprocess.STDOUT) # 判断子进程是否结束 while process.poll() is None: line = process.stdout.readline() @@ -81,34 +84,38 @@ def download_rclone(): download_url = asset["browser_download_url"] # rclone压缩包名 zip_name = asset["name"] - if os.path.exists(zip_name): - # 删除同名压缩包 - os.remove(zip_name) # 解压后目录名 dir_name = zip_name.replace(".zip", "") - if os.path.exists(dir_name): + + if not os.path.exists(zip_name): + # 删除同名压缩包 + # os.remove(zip_name) + + # 下载当前系统架构的文件 + # 创建一个opener对象 + opener = urllib.request.build_opener() + # 向opener传入请求头信息 + opener.addheaders.append(('User-agent', user_agent)) + # 将创建好的opener对象装入request + urllib.request.install_opener(opener) + filename, res = urllib.request.urlretrieve(download_url, zip_name) + # 从urlretrieve调用中清理临时文件 + urllib.request.urlcleanup() + if not os.path.exists(filename): + raise Exception("rclone下载失败！") + + if not os.path.exists(dir_name): # 删除同名目录，防止目录中的文件已被删除 - shutil.rmtree(dir_name) - # 下载当前系统架构的文件 - # 创建一个opener对象 - opener = urllib.request.build_opener() - # 向opener传入请求头信息 - opener.addheaders.append(('User-agent', user_agent)) - # 将创建好的opener对象装入request - urllib.request.install_opener(opener) - filename, res = urllib.request.urlretrieve(download_url, zip_name) - # 从urlretrieve调用中清理临时文件 - urllib.request.urlcleanup() - if not os.path.exists(filename): - raise Exception("rclone下载失败！") - if zipfile.is_zipfile(zip_name): - # 解压 - with zipfile.ZipFile(zip_name, "r") as zip_obj: - zip_obj.extractall(path=".") - # 授权：https://blog.csdn.net/u013632755/article/details/106599210 - # os.chmod(dir_name, stat.S_IRWXO + stat.S_IRWXO + stat.S_IRWXO + stat.S_IRWXO) - subprocess.run(['chmod', "-R", "777", dir_name], universal_newlines=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, shell=False) + # shutil.rmtree(dir_name) + + if zipfile.is_zipfile(zip_name): + # 解压 + with zipfile.ZipFile(zip_name, "r") as zip_obj: + zip_obj.extractall(path=".") + # 授权：https://blog.csdn.net/u013632755/article/details/106599210 + # os.chmod(dir_name, stat.S_IRWXO + stat.S_IRWXO + stat.S_IRWXO + stat.S_IRWXO) + run_cmd(['chmod', "-R", "777", dir_name], popen=False) + return dir_name @@ -335,13 +342,13 @@ def red_rclone_config(rclone_dir: str, name: str): return conf, file -def write_one_drive_config(rclone_dir: str, name: str, type="onedrive", drive_type="business", token=None, +def write_one_drive_config(rclone_dir: str, name: str, type_="onedrive", drive_type="business", token=None, drive_id=None): """ 此函数是为了方便写入在其他地方已经授权复制过来的One Drive配置，而不需要重新创建配置 :param rclone_dir: :param name: 自定义远程配置名称 - :param type: drive类型，一般默认即可 + :param type_: drive类型，一般默认即可 :param drive_type: :param token: 授权token :param drive_id: @@ -349,7 +356,7 @@ def write_one_drive_config(rclone_dir: str, name: str, type="onedrive", drive_ty """ conf, file = red_rclone_config(rclone_dir, name) - conf.set(name, 'type', type) + conf.set(name, 'type', type_) conf.set(name, 'drive_type', drive_type) # conf.set(name, 'region', "global") if token is not None: @@ -361,12 +368,14 @@ def write_one_drive_config(rclone_dir: str, name: str, type="onedrive", drive_ty conf.write(f) -def write_google_drive_config(rclone_dir: str, name: str, token=None, type="drive", scope="drive", team_drive=None, +def write_google_drive_config(rclone_dir: str, name: str, token=None, type_="drive", scope="drive", team_drive=None, root_folder_id=None, shared_with_me=None, service_account_file=None, saf=None): """ 此函数是为了方便写入在其他地方已经授权复制过来的Google Drive配置，而不需要重新创建配置 + :param rclone_dir: :param name: 自定义远程配置名称 :param token: 授权token + :param type_: :param drive_type: drive类型，一般默认即可 :param scope: rclone从驱动器请求访问时应使用的范围，对应--drive-scope参数 :param team_drive: 团队驱动器的ID，对应--drive-team-drive参数 @@ -378,7 +387,7 @@ def write_google_drive_config(rclone_dir: str, name: str, token=None, type="driv """ conf, file = red_rclone_config(rclone_dir, name) - conf.set(name, 'type', type) + conf.set(name, 'type', type_) conf.set(name, 'scope', scope) if token is not None: conf.set(name, 'token', token) @@ -453,12 +462,17 @@ def write_google_drive_config(rclone_dir: str, name: str, token=None, type="driv # 复制分享的链接文件或目录到团队盘 # run_cmd(f'./{rclone_dir}/rclone copy --drive-server-side-across-configs gdrive_stared: gdrive_team: {params}') + # 我的云盘同步到团队盘 # run_cmd(f'./{rclone_dir}/rclone sync --drive-server-side-across-configs gdrive: gdrive_team: {params}') + # 查看目录大小，可使用--drive-root-folder-id参数指定其他分享链接ID # run_cmd(f'./{rclone_dir}/rclone size gdrive_stared: ') + # 通过服务账户授权，指定账户查看网盘大小 # run_cmd(f'./{rclone_dir}/rclone -v --drive-impersonate woytu.com@gmail.com size gservicedrive: ') + +# 同步，排除参数--exclude "/{{images}}/" 只同步指定文件参数--include "/{{images}}/**" run_cmd(f'./{rclone_dir}/rclone sync gdrive:/ onedrive:/ {params}') # 去重 # run_cmd(f'./{rclone_dir}/rclone dedupe --dedupe-mode oldest gdrive:/ {params}')