没找到原因, 大佬们帮忙看看
项目使用 Django
+ uwsgi
+ Nginx
uwsgi
[uwsgi]
pythonpath=/usr/local/server
chdir=/home/server
env=DJANGO_SETTINGS_MODULE=conf.settings
module=server.wsgi
master=True
pidfile=logs/server.pid
vacuum=True
max-requests=1000
enable-threads=true
processes = 4
threads=8
listen=1024
daemonize=logs/wsgi.log
http=0.0.0.0:16020
buffer-size=32000
有一个从服务端获取 excel 文件的请求,请求全量 40 多万条数
响应时间过长, 然后浏览器得到Nginx 504
但是一个子进程的内存一直彪升
top -p 866
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
866 soe 20 0 7059m 5.3g 5740 S 100.8 33.9 4:17.34 uwsgi --ini /home/smb/work/soe_server
基本代码
from openpyxl.writer.excel import save_virtual_workbook
from django.http import HttpResponse
...
class ExcelTableObj(object):
def __init__(self, file_name=None):
if file_name:
self.file_name = file_name
self.wb = load_workbook(file_name)
else:
self.wb = Workbook()
def create_new_sheet(self, title='Sheet1'):
new_ws = self.wb.create_sheet(title=title)
def write_to_sheet(self, sheetname, datas, filename):
ws = self.wb[sheetname]
for data in datas:
ws.append(data)
self.wb.save(filename)
def update_sheet_name(self, sheetname):
ws = self.wb.active
ws.title = sheetname
def append_data_to_sheet(self, sheetname, data):
ws = self.wb[sheetname]
ws.append(data)
def save_file(self, file_name):
self.wb.save(file_name)
self.wb.close()
def get_upload_file_data(self, name=None):
if name:
ws = self.wb.get_sheet_by_name(name)
else:
ws = self.wb.worksheets[0]
rows = ws.max_row
cols = ws.max_column
file_data = []
fields = []
for i in range(1, cols+1):
cell = ws.cell(row=1, column=i)
if cell.value:
fields.append(cell.value.lower().strip())
for row in range(2, rows + 1):
row_data = {}
for j in range(len(fields)):
value = ws.cell(row=row, column=j+1).value
if value:
row_data[fields[j]] = str(value).strip()
if row_data:
file_data.append(row_data)
return file_data
def get_sheet_maxrow(self, name):
ws = self.wb.get_sheet_by_name(name)
rows = ws.max_row
return rows
def _get_download_data(datas):
for data in queryset :
...
item = [str(data.account_id),
ILLEGAL_CHARACTERS_RE.sub(r'', data.account_name) if data.account_name else data.account_name,
type, fb_aac_conf.FB_ACCOUNT_STATUS[data.account_status],
data.submitter, data.submit_time, data.confirmor, data.confirm_time,
fb_aac_conf.BATCH_STATUS[data.status], data.reason, data.entity_name, data.payment_name,
data.sale, data.ae_note, urgent
]
yield item
queryset = MyModel.objects.filter(...) # about `450k` rows
datas = _get_download_data(queryset)
excel = ExcelTableObj()
excel.update_sheet_name(sheetname)
excel.append_data_to_sheet(sheetname, title)
excel.write_to_sheet(sheetname, datas, filename)
excel.save_file(filename)
response = HttpResponse(save_virtual_workbook(excel.wb),
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
response['Content-Disposition'] = 'attachment; filename={}'.format(filename)
请大家帮忙分洗一下问题原因。感谢
这是一个专为移动设备优化的页面(即为了让你能够在 Google 搜索结果里秒开这个页面),如果你希望参与 V2EX 社区的讨论,你可以继续到 V2EX 上打开本讨论主题的完整版本。
V2EX 是创意工作者们的社区,是一个分享自己正在做的有趣事物、交流想法,可以遇见新朋友甚至新机会的地方。
V2EX is a community of developers, designers and creative people.