Попытка сохранить данные веб-сокета в CSV.Я могу получить первый ряд просто отлично, но когда он повторяется, чтобы добавить 2-й, он пытается преобразовать 1-й ряд снова, как он появляется.Я попытался выполнить сращивание, но не смог заставить его правильно изолировать последнюю строку.
- В прошлом я использовал to_datetime и работал нормально, когда извлекал данные и добавлял их в CSV.Не уверен, почему эта установка вызывает проблемы.
- Не могу правильно сгладить "k", если это бонус, который вы можете добавить в:)
Ошибка в терминале
0 2018-11-29 07:42:23.718000 ... XRPUSDT
1 1543477345793 ... XRPUSDT
[2 rows x 4 columns]
Unhandled Error
Traceback (most recent call last):
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\python\log.py", line 103, in callWithLogger
return callWithContext({"system": lp}, func, *args, **kw)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\python\log.py", line 86, in callWithContext
return context.call({ILogContext: newCtx}, func, *args, **kw)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\python\context.py", line 122, in callWithContext
return self.currentContext().callWithContext(ctx, func, *args, **kw)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\python\context.py", line 85, in callWithContext
return func(*args,**kw)
--- <exception caught here> ---
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\internet\selectreactor.py", line 149, in _doReadOrWrite
why = getattr(selectable, method)()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\internet\tcp.py", line 243, in doRead
return self._dataReceived(data)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\internet\tcp.py", line 249, in _dataReceived
rval = self.protocol.dataReceived(data)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\protocols\tls.py", line 330, in dataReceived
self._flushReceiveBIO()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\protocols\tls.py", line 295, in _flushReceiveBIO
ProtocolWrapper.dataReceived(self, bytes)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\twisted\protocols\policies.py", line 120, in dataReceived
self.wrappedProtocol.dataReceived(data)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\twisted\websocket.py", line 135, in dataReceived
self._dataReceived(data)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\websocket\protocol.py", line 1181, in _dataReceived
self.consumeData()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\websocket\protocol.py", line 1193, in consumeData
while self.processData() and self.state != WebSocketProtocol.STATE_CLOSED:
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\websocket\protocol.py", line 1559, in processData
fr = self.onFrameEnd()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\websocket\protocol.py", line 1681, in onFrameEnd
self._onMessageEnd()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\twisted\websocket.py", line 163, in _onMessageEnd
self.onMessageEnd()
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\websocket\protocol.py", line 628, in onMessageEnd
self._onMessage(payload, self.message_is_binary)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\autobahn\twisted\websocket.py", line 166, in _onMessage
self.onMessage(payload, isBinary)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\binance\websockets.py", line 32, in onMessage
self.factory.callback(payload_obj)
File "C:/Users/v-nademe/PycharmProjects/Crypto-Trader-MatplotLib/data_indicators/wss.py", line 86, in process_message
appendDFToCSV_void(df, "test.csv",",") # XRPUSD_INTERVAL_KRAKEN.CSV
File "C:/Users/v-nademe/PycharmProjects/Crypto-Trader-MatplotLib/data_indicators/wss.py", line 70, in appendDFToCSV_void
df['E'] = pd.to_datetime(df['E'], unit='ms')
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\pandas\core\tools\datetimes.py", line 451, in to_datetime
values = _convert_listlike(arg._values, True, format)
File "C:\Users\v-nademe\PycharmProjects\Crypto-Trader-MatplotLib\venv\lib\site-packages\pandas\core\tools\datetimes.py", line 303, in _convert_listlike
errors=errors)
File "pandas\_libs\tslib.pyx", line 339, in pandas._libs.tslib.array_with_unit_to_datetime
File "pandas\_libs\tslib.pyx", line 450, in pandas._libs.tslib.array_with_unit_to_datetime
builtins.ValueError: unit='ms' not valid with non-numerical val='2018-11-29 07:42:23.718000'
выход test.csv
2018-11-29 07:40:40.163,kline,"{'t': 1543477200000, 'T': 1543477259999, 's': 'XRPUSDT', 'i': '1m', 'f': 9664228, 'L': 9664250, 'o': '0.38050000', 'c': '0.38041000', 'h': '0.38060000', 'l': '0.38011000', 'v': '19550.60000000', 'n': 23, 'x': False, 'q': '7436.84001200', 'V': '8734.40000000', 'Q': '3323.36493600', 'B': '0'}",XRPUSDT
1543477255787.0,kline,"{'t': 1543477200000, 'T': 1543477259999, 's': 'XRPUSDT', 'i': '1m', 'f': 9664228, 'L': 9664279, 'o': '0.38050000', 'c': '0.38043000', 'h': '0.38060000', 'l': '0.38011000', 'v': '58424.00000000', 'n': 52, 'x': False, 'q': '22224.34474600', 'V': '43103.30000000', 'Q': '16398.17832500', 'B': '0'}",XRPUSDT
Формат данных
"""
{
"e": "kline", # event type
"E": 1499404907056, # event time
"s": "ETHBTC", # symbol
"k": {
"t": 1499404860000, # start time of this bar
"T": 1499404919999, # end time of this bar
"s": "ETHBTC", # symbol
"i": "1m", # interval
"f": 77462, # first trade id
"L": 77465, # last trade id
"o": "0.10278577", # open
"c": "0.10278645", # close
"h": "0.10278712", # high
"l": "0.10278518", # low
"v": "17.47929838", # volume
"n": 4, # number of trades
"x": false, # whether this bar is final
"q": "1.79662878", # quote volume
"V": "2.34879839", # volume of active buy
"Q": "0.24142166", # quote volume of active buy
"B": "13279784.01349473" # can be ignored
}
}
"""
def binance_xrpusdt_kline():
def appendDFToCSV_void(df, csvFilePath, sep=","):
import os
if not os.path.isfile(csvFilePath):
df['E'] = pd.to_datetime(df['E'], unit='ms')
df.to_csv(csvFilePath, mode='a', index=False, sep=sep)
elif len(df.columns) != len(pd.read_csv(csvFilePath, nrows=1, sep=sep, ).columns):
raise Exception(
"Columns do not match!! Dataframe has " + str(len(df.columns)) + " columns. CSV file has " + str(
len(pd.read_csv(csvFilePath, nrows=1, sep=sep).columns)) + " columns.")
elif not (df.columns == pd.read_csv(csvFilePath, nrows=1, sep=sep).columns).all():
raise Exception("Columns and column order of dataframe and csv file do not match!!")
else:
#print("preparing to save file to : " + str(symbol) + "_" + str(interval) + "_"+ str(exchange)+".csv")
df['E'] = pd.to_datetime(df['E'], unit='ms')
df.to_csv(csvFilePath, mode='a', index=False, sep=sep, header=False)
def process_message(msg):
global count, df, bm
pprint(msg)
#pprint("stream: {} data: {}".format(msg['e'], msg['E']))
# append message to array
df = df.append(msg, ignore_index=True)
#df = json_normalize(df['k'])
print(df.head())
df = pd.DataFrame(df)
appendDFToCSV_void(df, "test.csv",",") # XRPUSD_INTERVAL_KRAKEN.CSV
kline = bm.start_kline_socket('XRPUSDT', process_message)
bm.start()
binance_xrpusdt_kline()