python 串行执行和并行执行实例
作者:zhaoyangjian724 发布时间:2022-07-12 07:32:58
标签:python,串行,执行,并行
我就废话不多说了,大家还是直接看代码吧!
#coding=utf-8
import threading
import time
import cx_Oracle
from pprint import pprint
import csv
print time.asctime()
table_name = "dbtest.csv"
f = open(table_name + ".csv", "w")
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
def query01():
tname = threading.current_thread()
aa=10
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query02():
tname = threading.current_thread()
aa=20
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query03():
tname = threading.current_thread()
aa=30
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query04():
tname = threading.current_thread()
aa=40
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
def query05():
tname = threading.current_thread()
aa=50
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query06():
tname = threading.current_thread()
aa=60
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query07():
tname = threading.current_thread()
aa=70
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
return row
cursor.close()
def query08():
tname = threading.current_thread()
aa=80
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query09():
tname = threading.current_thread()
aa=90
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
def query10():
tname = threading.current_thread()
aa=100
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
threads = []
t1 = threading.Thread(target=query01, name='query01')
threads.append(t1)
t2 = threading.Thread(target=query02, name='query02')
threads.append(t2)
t2 = threading.Thread(target=query03, name='query03')
threads.append(t2)
t2 = threading.Thread(target=query04, name='query04')
threads.append(t2)
t2 = threading.Thread(target=query05, name='query05')
threads.append(t2)
t2 = threading.Thread(target=query06, name='query06')
threads.append(t2)
t2 = threading.Thread(target=query07, name='query07')
threads.append(t2)
t2 = threading.Thread(target=query08, name='query08')
threads.append(t2)
t2 = threading.Thread(target=query09, name='query09')
threads.append(t2)
t2 = threading.Thread(target=query10, name='query10')
threads.append(t2)
if __name__ == '__main__':
for t in threads:
# t.setDaemon(True)
t.start()
# t.run()
# t.start()
# print '3333333'
print threading.current_thread()
# print t.is_alive()
# print '3333333'
t.join()
print "all over "
print time.asctime()
C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py
Wed Mar 28 11:08:19 2018
<_MainThread(MainThread, started 18744)>
[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]
<_MainThread(MainThread, started 18744)>
[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]
<_MainThread(MainThread, started 18744)>
[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]
<_MainThread(MainThread, started 18744)>
[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]
<_MainThread(MainThread, started 18744)>
[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]
<_MainThread(MainThread, started 18744)>
[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]
<_MainThread(MainThread, started 18744)>
<_MainThread(MainThread, started 18744)>
[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]
<_MainThread(MainThread, started 18744)>
[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]
<_MainThread(MainThread, started 18744)>
[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]
all over
Wed Mar 28 11:08:34 2018
Process finished with exit code 0
这个时候是串行 花费了15秒
多线程跑:
#coding=utf-8
import threading
import time
import cx_Oracle
from pprint import pprint
import csv
print time.asctime()
table_name = "dbtest.csv"
f = open(table_name + ".csv", "w")
def query01():
tname = threading.current_thread()
aa=10
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query02():
tname = threading.current_thread()
aa=20
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query03():
tname = threading.current_thread()
aa=30
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query04():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=40
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query05():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=50
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query06():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=60
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query07():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=70
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
return row
cursor.close()
conn.close()
def query08():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=80
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query09():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=90
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
def query10():
conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')
cursor = conn.cursor()
tname = threading.current_thread()
aa=100
# cursor.execute("SELECT * FROM TEST100 WHERE ID = %s", [aa])
cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")
cursor.execute(None,{'id':aa})
row = cursor.fetchall()
print row
return row
cursor.close()
conn.close()
threads = []
t1 = threading.Thread(target=query01, name='query01')
threads.append(t1)
t2 = threading.Thread(target=query02, name='query02')
threads.append(t2)
t2 = threading.Thread(target=query03, name='query03')
threads.append(t2)
t2 = threading.Thread(target=query04, name='query04')
threads.append(t2)
t2 = threading.Thread(target=query05, name='query05')
threads.append(t2)
t2 = threading.Thread(target=query06, name='query06')
threads.append(t2)
t2 = threading.Thread(target=query07, name='query07')
threads.append(t2)
t2 = threading.Thread(target=query08, name='query08')
threads.append(t2)
t2 = threading.Thread(target=query09, name='query09')
threads.append(t2)
t2 = threading.Thread(target=query10, name='query10')
threads.append(t2)
if __name__ == '__main__':
for t in threads:
# t.setDaemon(True)
t.start()
# t.run()
# t.start()
# print '3333333'
print threading.current_thread()
# print t.is_alive()
# print '3333333'
t.join()
print "all over "
print time.asctime()
C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py
Wed Mar 28 11:12:47 2018
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
<_MainThread(MainThread, started 22500)>
[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]
[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]
[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]
[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]
[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]
[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]
[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]
[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]
[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]
all over
Wed Mar 28 11:12:55 2018
Process finished with exit code 0
此时花了 8秒
补充知识:python logging定制logstash的json日志格式
最近一直在折腾日志的收集,现在算是收尾了。 写一篇算python优化logstash的方案。
其实大家都知道logstash调用grok来解析日志的话,是要消耗cpu的成本的,毕竟是需要正则的匹配的。
根据logstash调优的方案,咱们可以预先生成json的格式。 我这边基本是python的程序,怎么搞尼 ?
有两种方法,第一种方法是生成json后,直接打入logstash的端口。 还有一种是生成json写入文件,让logstash做tail操作的时候,把一行的日志数据直接载入json就可以了。
python下的日志调试用得时logging,改成json也是很好改得。 另外不少老外已经考虑到这样的需求,已经做了python logstash的模块。
import logging
import logstash
import sys
host = 'localhost'
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
# test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))
test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')
# add extra field to logstash message
extra = {
'test_string': 'python version: ' + repr(sys.version_info),
'test_boolean': True,
'test_dict': {'a': 1, 'b': 'c'},
'test_float': 1.23,
'test_integer': 123,
'test_list': [1, 2, '3'],
}
test_logger.info('python-logstash: test extra fields', extra=extra)
python-logstash自带了amqp的方案
import logging
import logstash
# AMQP parameters
host = 'localhost'
username = 'guest'
password= 'guest'
exchange = 'logstash.py'
# get a logger and set logging level
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
# add the handler
test_logger.addHandler(logstash.AMQPLogstashHandler(version=1,
host=host,
durable=True,
username=username,
password=password,
exchange=exchange))
# log
test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')
try:
1/0
except:
test_logger.exception('python-logstash: test logstash exception with stack trace')
不管怎么说,最后生成的格式是这样就可以了。
{
"@source"=>"unknown",
"@type"=>"nginx",
"@tags"=>[],
"@fields"=>{
"remote_addr"=>"192.168.0.1",
"remote_user"=>"-",
"body_bytes_sent"=>"13988",
"request_time"=>"0.122",
"status"=>"200",
"request"=>"GET /some/url HTTP/1.1",
"request_method"=>"GET",
"http_referrer"=>"http://www.example.org/some/url",
"http_user_agent"=>"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1"
},
"@timestamp"=>"2012-08-23T10:49:14+02:00"
}
我这里简单提一下,这个模块用的不是很满意,我在python下把日志打成了json字符串,我原本以为会像grok那样,在Es里面,我的这条日志是个字段的结构,而不是这个日志都在message里面…. 我想大家应该明白了我的意思,这样很是不容易在kibana的搜索…
在kibana搜索,我经常上 source:xxx AND level:INFO 结果正像上面描述的那样,整条日志,都在@message里面。
来源:https://blog.csdn.net/zhaoyangjian724/article/details/79724956
0
投稿
猜你喜欢
- 对于大多数web应用来说,数据库都是一个十分基础性的部分。如果你在使用PHP,那么你很可能也在使用MySQL—LAMP系列中举足轻重的一份子
- 在后台添加中一般我们都设置了一个“关键字”,而且一般输入可能如下: 娃娃,毛绒娃娃,日本充气娃娃 我们将根据这个来做,具体实现如下: 先看一
- 如何取回已忘记的密码?forget.asp' 申请<html><head><title>闪亮日子
- 最近在学习Golang语言,中间遇到一个前辈指点,有一个学习原则:Learning By Doing。跟我之前学习Java的经验高度契合。在
- 一般情况下,访问或设置剪贴板,IE 只需使用 window.clipboardData 的 getData 或 setData 方法即可。M
- 内容摘要:本文介绍了通过获取访问者的IP地址来统计在线人数的方法,本文只是给出了实现统计在线人数的方法思路,具体代码的实现过程还得自己动手(
- 在Windows平台上,从原来使用C/C++编写原生EXE程序,到使用Python编写一些常用脚本程序,成熟的模块的使用使得编程效率大大提高
- 最近看到一个内部项目的插件加载机制,非常赞。当然这里说的插件并不是指的golang原生的可以在buildmode中加载指定so文件的那种加载
- 前言前面写过一篇用Python制作PPT的博客,感兴趣的可以参考用Python制作PPT这篇是关于用Python进行数据可视化的,准备作为一
- onactivate
- 问题:1. 访问 ASP 页面时,出现以下错误:Active Server Pages 错误 'ASP 0201'错误无效的
- 概要:Oracle关系数据库系统以其卓越的性能获得了广泛的应用,而保证数据库的安全性 是数据库管理工作的重要内容。本文是笔者在总结Oracl
- 每个进行过较大型的ASP-Web应用程序设计的开发人员大概都有如下的经历:ASP代码与页面HTML混淆难分,业务逻辑与显示方式绞合,使得代码
- 1。建立数据库表 表名为online 设如下字段 id '用来记录每一个访问都的session.sessionid name
- 一、浏览器允许每个域名所包含的 cookie 数:Microsoft 指出 Internet Explorer 8 增加 cookie 限制
- 第一次写这类文章,有点儿紧张有点儿新奇有点儿痛苦,来CDC实习2个月啦,每天除了工作就是体验体验再体验,因为之前做了一些有关规范的工作,突然
- 为什么需要优雅关闭什么叫优雅关闭?先说不优雅关闭,就是什么都不管,强制关闭进程,这会导致有些正在处理中的请求被强行中断这样做有什么问题?用户
- Win7或Windows server 2008中IIS7支持ASP+Access解决方法:1. 让IIS7支持ASPWin7或Window
- 介绍godep是解决包依赖的管理工具,目前最主流的一种,原理是扫描记录版本控制的信息,并在go命令前加壳来做到依赖管理godep 建议在 g
- 我们在使用bootstraptable做表格展示时,有时需要固定表格的高度当数据超出高度会出现滚动条,这时有可能出现表头列和数据列对不齐。出