私は次のようにユーザーごとに値を生成しています:
loDf = locDfs[user] # locDfs is a copy of locationDf elsewhere in the code... sorry for all the variable names.
loDf.reset_index(inplace=True)
loDf = pd.crosstab([loDf.date, loDf.uid], loDf.location_id)
loDf.reset_index(inplace=True)
loDf.set_index('date', inplace=True)
loDf.drop('uid', axis=1, inplace=True)
# join the location crosstab columns with the app crosstab columns per user
userLocAppDfs[user] = userLocAppDfs[user].join(loDf, how='outer')
# convert from just "1" at each location change event followed by zeros, to "1" continuing until next location change
userLocAppDfs[user] = userLocAppDfs[user].resample('15T').agg('max')
userLocAppDfs[user]['uid'].fillna(user, inplace=True)
これにより、位置データが取得され、location_idが列に変換され、時系列で他のデータと結合されます。
それは本質的にデータの再形成をカバーしています。次に、正規化する必要があります。これを行うには、各列の値を確認する必要があります。
for loc in locationDf[(locationDf['location_id'] in loDf.columns.values) & (locationDf['uid'] == user)].location_id.unique():
完全な機能は次のとおりです。
def normalize(inputMetricDf, inputLocationDf):
'''
normalize, resample, and combine data into a single data source
'''
metricDf = inputMetricDf.copy()
locationDf = inputLocationDf.copy()
appDf = metricDf[['date', 'uid', 'app_id', 'metric']].copy()
locDf = locationDf[['date', 'uid', 'location_id']]
locDf.set_index('date', inplace=True)
# convert location data to "15 minute interval" rows
locDfs = {}
for user, user_loc_dc in locDf.groupby('uid'):
locDfs[user] = user_loc_dc.resample('15T').agg('max').bfill()
aDf = appDf.copy()
aDf.set_index('date', inplace=True)
userLocAppDfs = {}
user = ''
for uid, a2_df in aDf.groupby('uid'):
user = uid
# per user, convert app data to 15m interval
userDf = a2_df.resample('15T').agg('max')
# assign metric for each app to an app column for each app, per user
userDf.reset_index(inplace=True)
userDf = pd.crosstab(index=userDf['date'], columns=userDf['app_id'],
values=userDf['metric'], aggfunc=np.mean).fillna(np.nan, downcast='infer')
userDf['uid'] = user
userDf.reset_index(inplace=True)
userDf.set_index('date', inplace=True)
# reapply 15m intervals now that we have new data per app
userLocAppDfs[user] = userDf.resample('15T').agg('max')
# assign location data to location columns per location, creates a "1" at the 15m interval of the location change event in the location column created
loDf = locDfs[user]
loDf.reset_index(inplace=True)
loDf = pd.crosstab([loDf.date, loDf.uid], loDf.location_id)
loDf.reset_index(inplace=True)
loDf.set_index('date', inplace=True)
loDf.drop('uid', axis=1, inplace=True)
# join the location crosstab columns with the app crosstab columns per user
userLocAppDfs[user] = userLocAppDfs[user].join(loDf, how='outer')
# convert from just "1" at each location change event followed by zeros, to "1" continuing until next location change
userLocAppDfs[user] = userLocAppDfs[user].resample('15T').agg('max')
userLocAppDfs[user]['uid'].fillna(user, inplace=True)
for loc in locationDf[(locationDf['location_id'] in loDf.columns.values) & (locationDf['uid'] == user)].location_id.unique():
# fill location NaNs
userLocAppDfs[user][loc] = userLocAppDfs[user][loc].replace(
np.nan, 0)
# fill app NaNs
for app in a2_df['app_id'].unique():
userLocAppDfs[user][app].interpolate(
method='linear', limit_area='inside', inplace=True)
userLocAppDfs[user][app].fillna(value=0, inplace=True)
df = userLocAppDfs[user].copy()
# ensure actual normality
alpha = 0.05
for app in aDf['app_id'].unique():
_, p = normaltest(userLocAppDfs[user][app])
if(p > alpha):
raise DataNotNormal(args=(user, app))
# for loc in userLocAppDfs[user]:
# could also test location data
return df
しかし、これはエラーを生成します:
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 346, in run_http_function
result = _function_handler.invoke_user_function(flask.request)
File "/env/local/lib/python3.7/site-packages/google/cloud/functions/worker.py", line 223, in invoke_user_function
loop.run_until_complete(future)
File "/opt/python3.7/lib/python3.7/asyncio/base_events.py", line 573, in run_until_complete
return future.result()
File "/user_code/main.py", line 31, in default_model
train, endog, exog, _, _, rawDf = preprocess(ledger, apps)
File "/user_code/Wrangling.py", line 67, in preprocess
rawDf = normalize(appDf, locDf)
File "/user_code/Wrangling.py", line 185, in normalize
for loc in locationDf[(locationDf['location_id'] in loDf.columns.values) & (locationDf['uid'] == user)].location_id.unique():
File "/env/local/lib/python3.7/site-packages/pandas/core/ops.py", line 1745, in wrapper
raise ValueError('Lengths must match to compare')
ValueError: Lengths must match to compare
形を変えたためにlocationsDfから場所が失われる可能性があることに気付く前は、次のことを行っていました。
for loc in locationDf[locationDf['uid'] == user].location_id.unique():
そしてこれは働いていた他のすべてのインスタンスに対して。しかし、同じ15tタイムセグメントに2つの場所があり、そのうちの1つがそこにしか表示されないが、15tウィンドウのためにドロップされた場合、エラーが発生します。だから私は他の条件が必要です。
locationDf ['location_id']は、クロス集計の列名と同じように、単なる文字列です。
なぜこれが私にエラーを投げるのですか?
for loc in locationDf[(locationDf['location_id'].isin(loDf.columns.values)) & (locationDf['uid'].isin([user])), 'location_id'].unique():
File "/env/local/lib/python3.7/site-packages/pandas/core/frame.py", line 2927, in __getitem__
indexer = self.columns.get_loc(key)
File "/env/local/lib/python3.7/site-packages/pandas/core/indexes/base.py", line 2657, in get_loc
return self._engine.get_loc(key)
File "pandas/_libs/index.pyx", line 108, in pandas._libs.index.IndexEngine.get_loc
File "pandas/_libs/index.pyx", line 110, in pandas._libs.index.IndexEngine.get_loc
TypeError: '(0 True
1 True
2 True
3 False
4 True
5 True
6 False
7 True
8 True
9 False
10 False
11 False
12 False
13 False
14 False
15 False
16 False
17 False
18 False
19 True
20 True
21 True
22 True
23 True
24 True
25 True
26 False
27 True
28 True
29 False
...
210 False
211 False
212 False
213 False
214 False
215 False
216 False
217 False
218 True
219 True
220 False
221 False
222 False
223 False
224 False
225 False
226 True
227 False
228 True
229 False
230 False
231 True
232 False
233 True
234 False
235 False
236 False
237 True
238 False
239 False
Length: 240, dtype: bool, 'location_id')' is an invalid key
条件を(を使用してisin
)に変更します
locationDf.loc[(locationDf['location_id'].isin(loDf.columns.values))
& (locationDf['uid'].isin(user)),'location_id'].unique()
更新
con1 = (locationDf['location_id'].isin(loDf.columns.values)
con2 = (locationDf['uid'].isin(pd.Series(user))
locationDf.loc[con1&con2,'location_id'].unique()
この記事はインターネットから収集されたものであり、転載の際にはソースを示してください。
侵害の場合は、連絡してください[email protected]
コメントを追加