You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
68 KiB
68 KiB
페키지 로드¶
In [1]:
%reload_ext watermark %watermark -v -p numpy,matplotlib,pandas,sklearn,tqdm,tensorflow,rpy2,watermark,feature_engine
CPython 3.6.9 IPython 7.16.2 numpy 1.19.5 matplotlib 3.3.4 pandas 1.1.5 sklearn 0.24.2 tqdm 4.62.3 tensorflow 2.6.2 rpy2 3.4.5 watermark 2.0.2 feature_engine 1.2.0
In [2]:
from rpy2.robjects import pandas2ri from rpy2.robjects import r from rpy2.robjects.packages import importr pandas2ri.activate() utils = importr('utils') package_names = ('ranger') utils.chooseCRANmirror(ind=1) #utils.install_packages("ranger") # ranger 패키지 설치
Out[2]:
<rpy2.rinterface_lib.sexp.NULLType object at 0x7f38e09093c8> [RTYPES.NILSXP]
In [3]:
import numpy as np import pandas as pd import os, re, cv2 from tqdm.auto import tqdm import matplotlib.pyplot as plt from feature_engine import transformation as vt from sklearn.model_selection import train_test_split import tensorflow as tf os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"]="1" main_dir="/root/data/dacon/open"
함수 생성¶
In [4]:
# 색 강조 def img_Contrast(img,clipLimit=3.0,tileGridSize=(8,8)): lab=cv2.cvtColor(img, cv2.COLOR_BGR2LAB) l,a,b=cv2.split(lab) clahe=cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize) cl=clahe.apply(l) limg=cv2.merge((cl,a,b)) final = cv2.cvtColor(limg,cv2.COLOR_LAB2BGR) return final # 특정범위 색 추출 def img_extract(return_img,img,lower=(0,0,0), upper=(110,255,200)): img_hsv=cv2.cvtColor(img, cv2.COLOR_BGR2HSV) img_mask=cv2.inRange(img_hsv, lower, upper) img_result= cv2.bitwise_and(return_img, return_img, mask=img_mask) img_hsv=cv2.cvtColor(img, cv2.COLOR_HSV2RGB) return img_result # 파일 목록 def file_list(directory): def find_files(directory): return([f"{directory}/{i}" \ for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)]) out=list() if type(directory)==str: out=find_files(directory) elif type(directory)==list: for folder in range(len(directory)): [out.append(file) for file in find_files(directory[folder])] return( sorted(out)) # 이미지 통계량 추출 def rgb_stat(img): r_m,g_m,b_m =np.mean(img,axis=(0,1)) r_sd,g_sd,b_sd= np.std(img,axis=(0,1)) return r_m,g_m,b_m,r_sd,g_sd,b_sd # 무게 산출 def img_to_weight(img,n=15000): return (cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)!=0).sum()/n # 파일 목록 def file_list(directory): def find_files(directory): return([f"{directory}/{i}" \ for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)]) out=list() if type(directory)==str: out=find_files(directory) elif type(directory)==list: for folder in range(len(directory)): [out.append(file) for file in find_files(directory[folder])] return( sorted(out)) # 이미지 변수 생성 def img_feature(dirs): df=pd.DataFrame({'img_dirs':dirs}) for i in tqdm(range(len(dirs))): img=cv2.imread(dirs[i]) raw_img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) # 시든 잎 추출 extract_img=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) df.loc[i,'del_leaf']=img_to_weight(extract_img,16900) # 청경채 추출 extract_img=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 df.loc[i,'pred_leaf1']=img_to_weight(extract_img,16900) contrast_img=img_Contrast(img, 3, tileGridSize=(5,5)) extract_img=img_extract(img,contrast_img,upper=(77,255,130))# 원본에서 추출 df.loc[i,'pred_leaf2']=img_to_weight(extract_img,16900) df.loc[i,'pred_leaf_mean']=(df.loc[i,'pred_leaf1']+df.loc[i,'pred_leaf2'])/2 #RGB 추출 df.loc[i,["r_m","g_m","b_m","r_sd","g_sd","b_sd"]]=rgb_stat(extract_img) return df
자료생성¶
In [5]:
tr_directory=[f"{main_dir}/train/CASE{i:02d}/image" for i in range(1,76)] tr_img_dirs=file_list(tr_directory) tr_img_dirs.remove(f'{main_dir}/train/CASE45/image/CASE45_17.png') te_img_dirs=file_list(f"{main_dir}/test/image") label_dfs=list() for file in tqdm([f"{main_dir}/train/CASE{i:02d}/label.csv" for i in range(1,76)]): temp_df=pd.read_csv(file) for i, img_file in enumerate(temp_df.img_name): time_df=pd.read_csv( f"{main_dir}/train/CASE{img_file[4:6]}/meta/{img_file.replace('jpg','png').replace('png','csv')}") time_df=time_df.sort_values('시간') time=time_df.loc[0,'시간'] temp_df.loc[i,'date']=pd.to_datetime(time).date() label_dfs.append(temp_df) label_df=pd.concat(label_dfs) label_df['case']=[i[:6] for i in label_df['img_name']] merge_df=label_df.copy() merge_df.columns=['img_name','now_weight','date','case'] merge_df=merge_df.drop("img_name",axis=1) merge_df['date']=merge_df.date+pd.to_timedelta(1,unit='day') label_df=pd.merge(label_df,merge_df,how='left',on=['case','date']) del merge_df
0%| | 0/75 [00:00<?, ?it/s]
In [6]:
if not('tr_df.csv' in os.listdir("/root/jupyter/데이콘/청경채/input/")): tr_df=img_feature(tr_img_dirs) te_df=img_feature(te_img_dirs) tr_df.to_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv',index=False) te_df.to_csv('/root/jupyter/데이콘/청경채/input/te_df.csv',index=False) else: tr_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv') te_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/te_df.csv') #라벨 for i in tqdm(range(tr_df.shape[0])): tr_df.loc[i,['leaf_weight','date','now_weight']]=label_df.loc[ label_df.img_name==tr_df.img_dirs[i].split('/')[-1],['leaf_weight','date','now_weight']].values[0]
0%| | 0/1591 [00:00<?, ?it/s]
삭제할 자료¶
CASE 2_10, 2_11, 34_01, 40_01, 40_02, 44_01, 52_01, 56_01, 60_20~34, 63_01, 64_01 : 환경자료 결측
CASE 8, 9, 22, 23, 26, 30, 31, 49, 59, 71, 72, 73 : 환경자료 결측
CASE 35_01, 41_01, 44_02, 45_01, 52_02, 53_01, 56_02, 57_01, 63_02 : 부분결측(제거)
CASE 34, 35, 48 : EC 결측
CASE 32_15, 51_11 : Co2 이상
In [7]:
env_na_p=[f"CASE{i}" for i in set([f"60_{i}" for i in range(20,34+1)]).union( set(['02_10','02_11','34_01','40_01','40_02','44_01','52_01','56_01','63_01','64_01']))] env_na_a=[f"CASE{i}" for i in ['08','09','22','23','26','30','31','49','59','71','72','73']] partial_na=[f"CASE{i}" for i in ["35_01","41_01","44_02","45_01","52_02","53_01","57_01","63_02"]] ec_na=[f"CASE{i}" for i in ["34","35","48"]] tr_df['na_label']=False for i in (env_na_p+env_na_a+partial_na+ec_na): tr_df.loc[tr_df['img_dirs'].str.contains(i),"na_label"]=True
In [8]:
for i, filename in tqdm(enumerate(tr_df.img_dirs)): temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv')) time_df=time_df.sort_values('시간') temp_df.시간=pd.to_datetime(temp_df.시간) temp_df['청색광추정광량']=(temp_df['총추정광량']- temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()] aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5) night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5) co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계 zero_ec_cnt=sum(temp_df['EC관측치']==0) disease_signal=co2_ratio*zero_ec_cnt aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5) night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5) night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5) m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True) m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True) # 결측자료 처리 if np.isnan(aftn_ec): if tr_df['del_leaf'][i]>10: aftn_ec=night_ec1*1.5 else: aftn_ec=night_ec2 if np.isnan(m_temp): m_temp=temp_df['내부온도관측치'].mean(skipna=True) if np.isnan(m_humidity): m_humidity=temp_df['내부습도관측치'].mean(skipna=True) if np.isnan(night_ec1): night_ec1=0 if np.isnan(co2_ratio): if tr_df.pred_leaf_mean[i]>50: co2_ratio = 0.5 else: co2_ratio = 1.5 disease_signal=co2_ratio*zero_ec_cnt tr_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal', 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\ co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity
0it [00:00, ?it/s]
In [9]:
for i, filename in tqdm(enumerate(te_df.img_dirs)): temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv')) time_df=time_df.sort_values('시간') temp_df.시간=pd.to_datetime(temp_df.시간) temp_df['청색광추정광량']=(temp_df['총추정광량']- temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()] aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5) night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5) co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계 zero_ec_cnt=sum(temp_df['EC관측치']==0) aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5) night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5) night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5) m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True) m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True) # 결측자료 처리 if np.isnan(aftn_ec): if te_df['del_leaf'][i]>10: aftn_ec=night_ec1*1.5 else: aftn_ec=night_ec2 if np.isnan(m_temp): m_temp=temp_df['내부온도관측치'].mean(skipna=True) if np.isnan(m_humidity): m_humidity=temp_df['내부습도관측치'].mean(skipna=True) if np.isnan(night_ec1): night_ec1=0 if np.isnan(co2_ratio): if te_df.pred_leaf_mean[i]>50: co2_ratio = 0.5 else: co2_ratio = 1.5 disease_signal=co2_ratio*zero_ec_cnt te_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal', 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\ co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity
0it [00:00, ?it/s]
청경채 무게 추정¶
In [10]:
df=tr_df[['img_dirs','leaf_weight']].dropna().reset_index() train,valid=train_test_split(df, test_size=0.33, random_state=42)
In [11]:
df[['img_dirs','leaf_weight']].isna().sum()
Out[11]:
img_dirs 0 leaf_weight 0 dtype: int64
CNN만으로 예측¶
In [12]:
def tr_gen(): df=train[['img_dirs','leaf_weight']].dropna().reset_index() for i in range(df.shape[0]): img=cv2.imread(df.img_dirs[i]) img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1) yield (extract_img/255, df.leaf_weight[i]) def va_gen(): df=valid[['img_dirs','leaf_weight']].dropna().reset_index() for i in range(df.shape[0]): img=cv2.imread(df.img_dirs[i]) img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1) yield (extract_img/255, df.leaf_weight[i]) def check_gen(): df=tr_df[['img_dirs','leaf_weight']].dropna().reset_index() for i in range(df.shape[0]): img=cv2.imread(df.img_dirs[i]) img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1) yield (extract_img/255, df.leaf_weight[i]) def te_gen(): df=te_df[['img_dirs']].reset_index() for i in range(df.shape[0]): img=cv2.imread(df.img_dirs[i]) img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1) yield (extract_img/255, np.nan) def NMAE(true, pred): mae = np.mean(np.abs(true-pred)) score = mae / np.mean(np.abs(true)) return score def nmae_keras(y_true, y_pred): score = tf.py_function(func=NMAE, inp=[y_true, y_pred], Tout=tf.float32, name='name') return score
In [16]:
tr_data=tf.data.Dataset.from_generator(tr_gen,(tf.float32,tf.float32)) tr_data=tr_data.cache().batch(12).prefetch(buffer_size=10) va_data=tf.data.Dataset.from_generator(va_gen,(tf.float32,tf.float32)) va_data=va_data.cache().batch(12).prefetch(buffer_size=10) te_data=tf.data.Dataset.from_generator(te_gen,(tf.float32,tf.float32)) te_data=te_data.cache().batch(12).prefetch(buffer_size=10) ch_data=tf.data.Dataset.from_generator(check_gen,(tf.float32,tf.float32)) ch_data=ch_data.cache().batch(12).prefetch(buffer_size=10)
In [17]:
next(iter(tr_data))[1]
Out[17]:
<tf.Tensor: shape=(12,), dtype=float32, numpy=
array([ 63.277, 0.621, 15.796, 6.407, 35.622, 17.865, 203.625,
11.203, 23.454, 0.291, 136.669, 59.01 ], dtype=float32)>
In [18]:
if not('forecast_weight_best_model_v4.h5' in os.listdir("/root/jupyter/데이콘/청경채/output/")): tf.random.set_seed(42) inp = tf.keras.Input(shape=(820, 616, 3),dtype=tf.float32) conv_1=tf.keras.layers.Conv2D(16,kernel_size=1, activation='LeakyReLU')(inp) avg_1=tf.keras.layers.AveragePooling2D()(conv_1) conv_2=tf.keras.layers.Conv2D(64,kernel_size=1, activation='LeakyReLU')(avg_1) avg_2=tf.keras.layers.AveragePooling2D()(conv_2) conv_3=tf.keras.layers.Conv2D(32,kernel_size=1, activation='LeakyReLU')(avg_2) avg_3=tf.keras.layers.AveragePooling2D()(conv_3) conv_4=tf.keras.layers.Conv2D(8,kernel_size=1, activation='LeakyReLU')(avg_3) avg_4=tf.keras.layers.AveragePooling2D()(conv_4) flat=tf.keras.layers.Flatten()(avg_4) dense_1=tf.keras.layers.Dense(64,activation='ReLU')(flat) dense_2=tf.keras.layers.Dense(32,activation='LeakyReLU')(dense_1) out=tf.keras.layers.Dense(1,activation='LeakyReLU')(dense_2) model = tf.keras.Model(inp, out) early = tf.keras.callbacks.EarlyStopping( monitor='val_loss',mode="min", patience=10) lr_reduce=tf.keras.callbacks.ReduceLROnPlateau( monitor='val_loss',patience=3,verbose=1,min_delta=0.001) model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01), loss='mse',metrics=['mae']) model.fit(tr_data,verbose=1,callbacks =[early,lr_reduce], epochs=50,validation_data=va_data) # fine tuning for i in model.layers[0:-4]: i.trainable=False cp_callback = tf.keras.callbacks.ModelCheckpoint( filepath='/root/jupyter/데이콘/청경채/model4/now_weight_{val_loss:.2f}.h5', monitor='val_loss',mode='min',verbose=1) early = tf.keras.callbacks.EarlyStopping(monitor='val_loss',mode="min", patience=50) lr_reduce=tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',patience=10,verbose=1) tf.random.set_seed(42) model.fit(tr_data,verbose=1,callbacks =[early,lr_reduce,cp_callback], epochs=300,validation_data=va_data) file_loss=min([i.split('/')[-1].split('_')[-1].replace('.h5','') for i in os.listdir('/root/jupyter/데이콘/청경채/model4/')]) model=tf.keras.models.load_model(f'/root/jupyter/데이콘/청경채/model4/now_weight_{file_loss}.h5') tf.keras.models.save_model(model,f'/root/jupyter/데이콘/청경채/output/forecast_weight_best_model_v4.h5') else: model=tf.keras.models.load_model(f'/root/jupyter/데이콘/청경채/output/forecast_weight_best_model_v4.h5')
Epoch 1/50 89/89 [==============================] - 178s 2s/step - loss: 2706.1069 - mae: 32.4218 - val_loss: 1184.8112 - val_mae: 21.1455 Epoch 2/50 89/89 [==============================] - 16s 178ms/step - loss: 1075.7421 - mae: 17.7971 - val_loss: 605.5364 - val_mae: 15.9050 Epoch 3/50 89/89 [==============================] - 16s 179ms/step - loss: 1062.9193 - mae: 18.7751 - val_loss: 769.4287 - val_mae: 13.2192 Epoch 4/50 89/89 [==============================] - 16s 179ms/step - loss: 945.1603 - mae: 17.7024 - val_loss: 367.3741 - val_mae: 8.4145 Epoch 5/50 89/89 [==============================] - 16s 179ms/step - loss: 877.6848 - mae: 16.4986 - val_loss: 401.5463 - val_mae: 9.7904 Epoch 6/50 89/89 [==============================] - 16s 179ms/step - loss: 817.7131 - mae: 16.0758 - val_loss: 260.6157 - val_mae: 8.5514 Epoch 7/50 89/89 [==============================] - 16s 180ms/step - loss: 378.7823 - mae: 10.6879 - val_loss: 232.7030 - val_mae: 7.4931 Epoch 8/50 89/89 [==============================] - 16s 181ms/step - loss: 289.2178 - mae: 9.4165 - val_loss: 218.2774 - val_mae: 7.5454 Epoch 9/50 89/89 [==============================] - 16s 182ms/step - loss: 215.8115 - mae: 8.0182 - val_loss: 209.4668 - val_mae: 7.4752 Epoch 10/50 89/89 [==============================] - 16s 182ms/step - loss: 161.7319 - mae: 6.9298 - val_loss: 194.4687 - val_mae: 6.8856 Epoch 11/50 89/89 [==============================] - 16s 182ms/step - loss: 131.0336 - mae: 6.2625 - val_loss: 193.1138 - val_mae: 6.8111 Epoch 12/50 89/89 [==============================] - 16s 182ms/step - loss: 111.2545 - mae: 5.7628 - val_loss: 178.7514 - val_mae: 6.4462 Epoch 13/50 89/89 [==============================] - 16s 183ms/step - loss: 96.7691 - mae: 5.3750 - val_loss: 172.1933 - val_mae: 6.4611 Epoch 14/50 89/89 [==============================] - 16s 184ms/step - loss: 86.9064 - mae: 5.1046 - val_loss: 168.3996 - val_mae: 6.2891 Epoch 15/50 89/89 [==============================] - 16s 184ms/step - loss: 78.1423 - mae: 4.8621 - val_loss: 166.2413 - val_mae: 6.3767 Epoch 16/50 89/89 [==============================] - 16s 184ms/step - loss: 74.6942 - mae: 4.8733 - val_loss: 164.6711 - val_mae: 6.2652 Epoch 17/50 89/89 [==============================] - 16s 183ms/step - loss: 66.9943 - mae: 4.5377 - val_loss: 162.0831 - val_mae: 6.1993 Epoch 18/50 89/89 [==============================] - 16s 184ms/step - loss: 67.0172 - mae: 4.5799 - val_loss: 178.3701 - val_mae: 6.6673 Epoch 19/50 89/89 [==============================] - 16s 184ms/step - loss: 61.7426 - mae: 4.4534 - val_loss: 154.2710 - val_mae: 6.0991 Epoch 20/50 89/89 [==============================] - 16s 185ms/step - loss: 57.2921 - mae: 4.2976 - val_loss: 178.2121 - val_mae: 6.3892 Epoch 21/50 89/89 [==============================] - 16s 184ms/step - loss: 53.3747 - mae: 4.1471 - val_loss: 171.2368 - val_mae: 6.3101 Epoch 22/50 89/89 [==============================] - 16s 184ms/step - loss: 57.1590 - mae: 4.3407 - val_loss: 152.3658 - val_mae: 6.0626 Epoch 23/50 89/89 [==============================] - 16s 184ms/step - loss: 64.3151 - mae: 4.6944 - val_loss: 169.2197 - val_mae: 6.4303 Epoch 24/50 89/89 [==============================] - 16s 183ms/step - loss: 64.4714 - mae: 4.7079 - val_loss: 172.1687 - val_mae: 5.9832 Epoch 25/50 89/89 [==============================] - 16s 185ms/step - loss: 113.6208 - mae: 5.9376 - val_loss: 190.5305 - val_mae: 7.3836 Epoch 00025: ReduceLROnPlateau reducing learning rate to 0.0009999999776482583. Epoch 26/50 89/89 [==============================] - 16s 185ms/step - loss: 98.9548 - mae: 5.6564 - val_loss: 140.8842 - val_mae: 5.3808 Epoch 27/50 89/89 [==============================] - 16s 185ms/step - loss: 67.3946 - mae: 4.4792 - val_loss: 131.1024 - val_mae: 5.1512 Epoch 28/50 89/89 [==============================] - 16s 183ms/step - loss: 57.2217 - mae: 4.1348 - val_loss: 125.3723 - val_mae: 5.0370 Epoch 29/50 89/89 [==============================] - 16s 185ms/step - loss: 51.4432 - mae: 3.9225 - val_loss: 121.3438 - val_mae: 4.9430 Epoch 30/50 89/89 [==============================] - 16s 183ms/step - loss: 47.5067 - mae: 3.7674 - val_loss: 118.2129 - val_mae: 4.8690 Epoch 31/50 89/89 [==============================] - 16s 184ms/step - loss: 44.6266 - mae: 3.6404 - val_loss: 115.5802 - val_mae: 4.8066 Epoch 32/50 89/89 [==============================] - 16s 185ms/step - loss: 42.3210 - mae: 3.5359 - val_loss: 113.2968 - val_mae: 4.7533 Epoch 33/50 89/89 [==============================] - 17s 186ms/step - loss: 40.5360 - mae: 3.4547 - val_loss: 111.4342 - val_mae: 4.7086 Epoch 34/50 89/89 [==============================] - 16s 184ms/step - loss: 38.9372 - mae: 3.3816 - val_loss: 109.7459 - val_mae: 4.6709 Epoch 35/50 89/89 [==============================] - 16s 184ms/step - loss: 37.5384 - mae: 3.3131 - val_loss: 108.2381 - val_mae: 4.6379 Epoch 36/50 89/89 [==============================] - 16s 185ms/step - loss: 36.3591 - mae: 3.2572 - val_loss: 106.9405 - val_mae: 4.6114 Epoch 37/50 89/89 [==============================] - 16s 185ms/step - loss: 35.2517 - mae: 3.2012 - val_loss: 105.8183 - val_mae: 4.5846 Epoch 38/50 89/89 [==============================] - 16s 185ms/step - loss: 34.2525 - mae: 3.1520 - val_loss: 104.7308 - val_mae: 4.5574 Epoch 39/50 89/89 [==============================] - 17s 186ms/step - loss: 33.3317 - mae: 3.1037 - val_loss: 103.8288 - val_mae: 4.5360 Epoch 40/50 89/89 [==============================] - 16s 185ms/step - loss: 32.4678 - mae: 3.0596 - val_loss: 103.0781 - val_mae: 4.5183 Epoch 41/50 89/89 [==============================] - 16s 185ms/step - loss: 31.5418 - mae: 3.0129 - val_loss: 102.3107 - val_mae: 4.4954 Epoch 42/50 89/89 [==============================] - 16s 185ms/step - loss: 30.8920 - mae: 2.9777 - val_loss: 101.7379 - val_mae: 4.4812 Epoch 43/50 89/89 [==============================] - 16s 185ms/step - loss: 30.1488 - mae: 2.9423 - val_loss: 101.2127 - val_mae: 4.4668 Epoch 44/50 89/89 [==============================] - 16s 185ms/step - loss: 29.3677 - mae: 2.9011 - val_loss: 100.6727 - val_mae: 4.4530 Epoch 45/50 89/89 [==============================] - 16s 185ms/step - loss: 28.6850 - mae: 2.8625 - val_loss: 100.3327 - val_mae: 4.4408 Epoch 46/50 89/89 [==============================] - 17s 185ms/step - loss: 28.0526 - mae: 2.8331 - val_loss: 99.9700 - val_mae: 4.4388 Epoch 47/50 89/89 [==============================] - 16s 185ms/step - loss: 27.4110 - mae: 2.8003 - val_loss: 99.6259 - val_mae: 4.4341 Epoch 48/50 89/89 [==============================] - 16s 184ms/step - loss: 26.7111 - mae: 2.7650 - val_loss: 99.2249 - val_mae: 4.4276 Epoch 49/50 89/89 [==============================] - 16s 185ms/step - loss: 26.0827 - mae: 2.7322 - val_loss: 98.9602 - val_mae: 4.4216 Epoch 50/50 89/89 [==============================] - 16s 185ms/step - loss: 25.5295 - mae: 2.7022 - val_loss: 98.7301 - val_mae: 4.4294 Epoch 1/300 89/89 [==============================] - 17s 186ms/step - loss: 24.9081 - mae: 2.6677 - val_loss: 98.4472 - val_mae: 4.4277 Epoch 00001: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.45.h5 Epoch 2/300 89/89 [==============================] - 17s 186ms/step - loss: 24.2711 - mae: 2.6347 - val_loss: 98.3397 - val_mae: 4.4309 Epoch 00002: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.34.h5 Epoch 3/300 89/89 [==============================] - 16s 184ms/step - loss: 23.7050 - mae: 2.6048 - val_loss: 98.1409 - val_mae: 4.4259 Epoch 00003: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.14.h5 Epoch 4/300 89/89 [==============================] - 16s 184ms/step - loss: 23.1324 - mae: 2.5787 - val_loss: 98.0273 - val_mae: 4.4341 Epoch 00004: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.03.h5 Epoch 5/300 89/89 [==============================] - 16s 184ms/step - loss: 22.4775 - mae: 2.5455 - val_loss: 97.7619 - val_mae: 4.4271 Epoch 00005: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.76.h5 Epoch 6/300 89/89 [==============================] - 16s 185ms/step - loss: 21.8488 - mae: 2.5157 - val_loss: 97.6119 - val_mae: 4.4255 Epoch 00006: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.61.h5 Epoch 7/300 89/89 [==============================] - 16s 185ms/step - loss: 21.4512 - mae: 2.4887 - val_loss: 97.6136 - val_mae: 4.4488 Epoch 00007: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.61.h5 Epoch 8/300 89/89 [==============================] - 16s 186ms/step - loss: 20.8273 - mae: 2.4549 - val_loss: 97.5571 - val_mae: 4.4427 Epoch 00008: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.56.h5 Epoch 9/300 89/89 [==============================] - 16s 185ms/step - loss: 20.3285 - mae: 2.4318 - val_loss: 97.6005 - val_mae: 4.4564 Epoch 00009: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.60.h5 Epoch 10/300 89/89 [==============================] - 16s 185ms/step - loss: 19.7302 - mae: 2.3989 - val_loss: 97.5921 - val_mae: 4.4574 Epoch 00010: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.59.h5 Epoch 11/300 89/89 [==============================] - 16s 185ms/step - loss: 19.1184 - mae: 2.3661 - val_loss: 97.5904 - val_mae: 4.4564 Epoch 00011: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.59.h5 Epoch 12/300 89/89 [==============================] - 16s 185ms/step - loss: 18.5368 - mae: 2.3331 - val_loss: 97.5407 - val_mae: 4.4435 Epoch 00012: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.54.h5 Epoch 13/300 89/89 [==============================] - 17s 186ms/step - loss: 17.9736 - mae: 2.2973 - val_loss: 97.6559 - val_mae: 4.4630 Epoch 00013: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.66.h5 Epoch 14/300 89/89 [==============================] - 16s 185ms/step - loss: 17.4061 - mae: 2.2623 - val_loss: 97.6568 - val_mae: 4.4542 Epoch 00014: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.66.h5 Epoch 15/300 89/89 [==============================] - 17s 186ms/step - loss: 16.8778 - mae: 2.2292 - val_loss: 97.7953 - val_mae: 4.4584 Epoch 00015: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.80.h5 Epoch 16/300 89/89 [==============================] - 16s 184ms/step - loss: 16.3516 - mae: 2.1968 - val_loss: 97.9856 - val_mae: 4.4684 Epoch 00016: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_97.99.h5 Epoch 17/300 89/89 [==============================] - 16s 184ms/step - loss: 15.9213 - mae: 2.1658 - val_loss: 98.1391 - val_mae: 4.4674 Epoch 00017: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.14.h5 Epoch 18/300 89/89 [==============================] - 16s 184ms/step - loss: 15.4090 - mae: 2.1306 - val_loss: 98.3820 - val_mae: 4.4780 Epoch 00018: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.38.h5 Epoch 19/300 89/89 [==============================] - 16s 182ms/step - loss: 14.8508 - mae: 2.0930 - val_loss: 98.7418 - val_mae: 4.4959 Epoch 00019: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.74.h5 Epoch 20/300 89/89 [==============================] - 16s 182ms/step - loss: 14.6505 - mae: 2.0827 - val_loss: 98.9809 - val_mae: 4.5037 Epoch 00020: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_98.98.h5 Epoch 21/300 89/89 [==============================] - 16s 182ms/step - loss: 13.9771 - mae: 2.0329 - val_loss: 99.5387 - val_mae: 4.5324 Epoch 00021: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.54.h5 Epoch 22/300 89/89 [==============================] - 16s 182ms/step - loss: 13.5490 - mae: 1.9979 - val_loss: 100.1497 - val_mae: 4.5425 Epoch 00022: ReduceLROnPlateau reducing learning rate to 9.999999310821295e-05. Epoch 00022: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_100.15.h5 Epoch 23/300 89/89 [==============================] - 16s 181ms/step - loss: 16.5462 - mae: 2.1990 - val_loss: 99.1078 - val_mae: 4.3856 Epoch 00023: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.11.h5 Epoch 24/300 89/89 [==============================] - 16s 181ms/step - loss: 15.0572 - mae: 2.0653 - val_loss: 99.1320 - val_mae: 4.3674 Epoch 00024: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.13.h5 Epoch 25/300 89/89 [==============================] - 16s 182ms/step - loss: 14.5310 - mae: 2.0305 - val_loss: 99.1846 - val_mae: 4.3557 Epoch 00025: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.18.h5 Epoch 26/300 89/89 [==============================] - 16s 181ms/step - loss: 14.1015 - mae: 2.0015 - val_loss: 99.2505 - val_mae: 4.3480 Epoch 00026: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.25.h5 Epoch 27/300 89/89 [==============================] - 16s 181ms/step - loss: 13.7482 - mae: 1.9756 - val_loss: 99.3067 - val_mae: 4.3421 Epoch 00027: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.31.h5 Epoch 28/300 89/89 [==============================] - 16s 180ms/step - loss: 13.4415 - mae: 1.9513 - val_loss: 99.3641 - val_mae: 4.3373 Epoch 00028: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.36.h5 Epoch 29/300 89/89 [==============================] - 16s 180ms/step - loss: 13.1685 - mae: 1.9294 - val_loss: 99.4337 - val_mae: 4.3348 Epoch 00029: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.43.h5 Epoch 30/300 89/89 [==============================] - 16s 181ms/step - loss: 12.9188 - mae: 1.9087 - val_loss: 99.4618 - val_mae: 4.3308 Epoch 00030: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.46.h5 Epoch 31/300 89/89 [==============================] - 16s 181ms/step - loss: 12.6920 - mae: 1.8901 - val_loss: 99.5223 - val_mae: 4.3293 Epoch 00031: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.52.h5 Epoch 32/300 89/89 [==============================] - 16s 180ms/step - loss: 12.4808 - mae: 1.8719 - val_loss: 99.5526 - val_mae: 4.3259 Epoch 00032: ReduceLROnPlateau reducing learning rate to 9.999999019782991e-06. Epoch 00032: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_99.55.h5 Epoch 33/300 89/89 [==============================] - 16s 181ms/step - loss: 15.1895 - mae: 2.0991 - val_loss: 101.8241 - val_mae: 4.1671 Epoch 00033: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_101.82.h5 Epoch 34/300 89/89 [==============================] - 16s 180ms/step - loss: 13.2418 - mae: 1.9099 - val_loss: 102.4766 - val_mae: 4.1610 Epoch 00034: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.48.h5 Epoch 35/300 89/89 [==============================] - 16s 181ms/step - loss: 13.1049 - mae: 1.8921 - val_loss: 102.5064 - val_mae: 4.1605 Epoch 00035: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.51.h5 Epoch 36/300 89/89 [==============================] - 16s 180ms/step - loss: 13.0506 - mae: 1.8870 - val_loss: 102.4847 - val_mae: 4.1608 Epoch 00036: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.48.h5 Epoch 37/300 89/89 [==============================] - 16s 180ms/step - loss: 13.0094 - mae: 1.8841 - val_loss: 102.4706 - val_mae: 4.1615 Epoch 00037: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.47.h5 Epoch 38/300 89/89 [==============================] - 16s 180ms/step - loss: 12.9733 - mae: 1.8816 - val_loss: 102.4666 - val_mae: 4.1624 Epoch 00038: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.47.h5 Epoch 39/300 89/89 [==============================] - 16s 180ms/step - loss: 12.9396 - mae: 1.8795 - val_loss: 102.4724 - val_mae: 4.1633 Epoch 00039: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.47.h5 Epoch 40/300 89/89 [==============================] - 16s 180ms/step - loss: 12.9082 - mae: 1.8773 - val_loss: 102.4831 - val_mae: 4.1642 Epoch 00040: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.48.h5 Epoch 41/300 89/89 [==============================] - 16s 180ms/step - loss: 12.8786 - mae: 1.8753 - val_loss: 102.4957 - val_mae: 4.1650 Epoch 00041: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.50.h5 Epoch 42/300 89/89 [==============================] - 16s 180ms/step - loss: 12.8502 - mae: 1.8733 - val_loss: 102.5135 - val_mae: 4.1658 Epoch 00042: ReduceLROnPlateau reducing learning rate to 9.99999883788405e-07. Epoch 00042: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.51.h5 Epoch 43/300 89/89 [==============================] - 16s 179ms/step - loss: 12.1413 - mae: 1.8241 - val_loss: 102.5824 - val_mae: 4.1665 Epoch 00043: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.58.h5 Epoch 44/300 89/89 [==============================] - 16s 179ms/step - loss: 12.1302 - mae: 1.8225 - val_loss: 102.6604 - val_mae: 4.1675 Epoch 00044: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.66.h5 Epoch 45/300 89/89 [==============================] - 16s 180ms/step - loss: 12.1223 - mae: 1.8212 - val_loss: 102.7269 - val_mae: 4.1684 Epoch 00045: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.73.h5 Epoch 46/300 89/89 [==============================] - 16s 180ms/step - loss: 12.1159 - mae: 1.8201 - val_loss: 102.7830 - val_mae: 4.1691 Epoch 00046: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.78.h5 Epoch 47/300 89/89 [==============================] - 16s 179ms/step - loss: 12.1107 - mae: 1.8192 - val_loss: 102.8303 - val_mae: 4.1698 Epoch 00047: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.83.h5 Epoch 48/300 89/89 [==============================] - 16s 179ms/step - loss: 12.1061 - mae: 1.8184 - val_loss: 102.8703 - val_mae: 4.1703 Epoch 00048: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.87.h5 Epoch 49/300 89/89 [==============================] - 16s 179ms/step - loss: 12.1022 - mae: 1.8177 - val_loss: 102.9039 - val_mae: 4.1708 Epoch 00049: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.90.h5 Epoch 50/300 89/89 [==============================] - 16s 179ms/step - loss: 12.0986 - mae: 1.8171 - val_loss: 102.9325 - val_mae: 4.1712 Epoch 00050: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.93.h5 Epoch 51/300 89/89 [==============================] - 16s 180ms/step - loss: 12.0953 - mae: 1.8166 - val_loss: 102.9565 - val_mae: 4.1715 Epoch 00051: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.96.h5 Epoch 52/300 89/89 [==============================] - 16s 180ms/step - loss: 12.0923 - mae: 1.8162 - val_loss: 102.9770 - val_mae: 4.1719 Epoch 00052: ReduceLROnPlateau reducing learning rate to 9.99999883788405e-08. Epoch 00052: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.98.h5 Epoch 53/300 89/89 [==============================] - 16s 181ms/step - loss: 12.0136 - mae: 1.8093 - val_loss: 102.9793 - val_mae: 4.1719 Epoch 00053: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.98.h5 Epoch 54/300 89/89 [==============================] - 16s 182ms/step - loss: 12.0134 - mae: 1.8092 - val_loss: 102.9818 - val_mae: 4.1719 Epoch 00054: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.98.h5 Epoch 55/300 89/89 [==============================] - 16s 182ms/step - loss: 12.0131 - mae: 1.8092 - val_loss: 102.9843 - val_mae: 4.1720 Epoch 00055: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.98.h5 Epoch 56/300 89/89 [==============================] - 16s 180ms/step - loss: 12.0128 - mae: 1.8091 - val_loss: 102.9867 - val_mae: 4.1720 Epoch 00056: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.99.h5 Epoch 57/300 89/89 [==============================] - 16s 181ms/step - loss: 12.0125 - mae: 1.8091 - val_loss: 102.9890 - val_mae: 4.1720 Epoch 00057: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.99.h5 Epoch 58/300 89/89 [==============================] - 16s 183ms/step - loss: 12.0122 - mae: 1.8091 - val_loss: 102.9913 - val_mae: 4.1721 Epoch 00058: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.99.h5 Epoch 59/300 89/89 [==============================] - 16s 183ms/step - loss: 12.0119 - mae: 1.8090 - val_loss: 102.9936 - val_mae: 4.1721 Epoch 00059: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_102.99.h5 Epoch 60/300 89/89 [==============================] - 16s 182ms/step - loss: 12.0117 - mae: 1.8090 - val_loss: 102.9959 - val_mae: 4.1721 Epoch 00060: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_103.00.h5 Epoch 61/300 89/89 [==============================] - 16s 182ms/step - loss: 12.0114 - mae: 1.8089 - val_loss: 102.9981 - val_mae: 4.1722 Epoch 00061: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_103.00.h5 Epoch 62/300 89/89 [==============================] - 16s 183ms/step - loss: 12.0111 - mae: 1.8089 - val_loss: 103.0003 - val_mae: 4.1722 Epoch 00062: ReduceLROnPlateau reducing learning rate to 9.999998695775504e-09. Epoch 00062: saving model to /root/jupyter/데이콘/청경채/model4/now_weight_103.00.h5
In [19]:
tr_df['cnn_now_weight']=model.predict(ch_data) te_df['cnn_now_weight']=model.predict(te_data)
In [21]:
plt.scatter(tr_df.cnn_now_weight,tr_df.leaf_weight)
Out[21]:
<matplotlib.collections.PathCollection at 0x7f38039b1940>
In [20]:
submit=pd.read_csv(f"{main_dir}/sample_submission.csv") submit['leaf_weight']=te_df['cnn_now_weight'] submit.to_csv(f"/root/jupyter/데이콘/청경채/output/submit_7.csv",index=False)