chinensis_v2/submit_8.ipynb

페키지 로드¶

In [1]:

%reload_ext watermark
%watermark -v -p numpy,matplotlib,pandas,sklearn,tqdm,tensorflow,rpy2,watermark,feature_engine

CPython 3.6.9
IPython 7.16.2

numpy 1.19.5
matplotlib 3.3.4
pandas 1.1.5
sklearn 0.24.2
tqdm 4.62.3
tensorflow 2.6.2
rpy2 3.4.5
watermark 2.0.2
feature_engine 1.2.0

In [2]:

from rpy2.robjects import pandas2ri
from rpy2.robjects import r
from rpy2.robjects.packages import importr
pandas2ri.activate()

utils = importr('utils')
package_names = ('ranger')
utils.chooseCRANmirror(ind=1)
#utils.install_packages("ranger") # ranger 패키지 설치

Out[2]:

<rpy2.rinterface_lib.sexp.NULLType object at 0x7f3359e8e288> [RTYPES.NILSXP]

In [3]:

import numpy as np
import pandas as pd
import os, re, cv2
from tqdm.auto import tqdm
import matplotlib.pyplot as plt
from feature_engine import transformation as vt
from sklearn.model_selection import train_test_split
import tensorflow as tf
from sklearn.preprocessing import MinMaxScaler
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" 
os.environ["CUDA_VISIBLE_DEVICES"]="0"
main_dir="/root/data/dacon/open"

함수 생성¶

In [4]:

# 색 강조
def img_Contrast(img,clipLimit=3.0,tileGridSize=(8,8)):
    lab=cv2.cvtColor(img, cv2.COLOR_BGR2LAB)
    l,a,b=cv2.split(lab)
    clahe=cv2.createCLAHE(clipLimit=clipLimit,
                          tileGridSize=tileGridSize)
    cl=clahe.apply(l)
    limg=cv2.merge((cl,a,b))
    final = cv2.cvtColor(limg,cv2.COLOR_LAB2BGR)
    return final

# 특정범위 색 추출
def img_extract(return_img,img,lower=(0,0,0), upper=(110,255,200)):
    img_hsv=cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    img_mask=cv2.inRange(img_hsv, lower, upper)
    img_result= cv2.bitwise_and(return_img, return_img, mask=img_mask)
    img_hsv=cv2.cvtColor(img, cv2.COLOR_HSV2RGB)
    return img_result

# 파일 목록
def file_list(directory):
    def find_files(directory):
        return([f"{directory}/{i}" \
                for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)])
    out=list()
    if type(directory)==str:
        out=find_files(directory)
    elif type(directory)==list:
        for folder in range(len(directory)):
            [out.append(file) for file in find_files(directory[folder])]
    return(
        sorted(out))

# 이미지 통계량 추출
def rgb_stat(img):
    r_m,g_m,b_m   =np.mean(img,axis=(0,1))
    r_sd,g_sd,b_sd= np.std(img,axis=(0,1))
    return r_m,g_m,b_m,r_sd,g_sd,b_sd

# 무게 산출
def img_to_weight(img,n=15000):
    return (cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)!=0).sum()/n

# 파일 목록
def file_list(directory):
    def find_files(directory):
        return([f"{directory}/{i}" \
                for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)])
    out=list()
    if type(directory)==str:
        out=find_files(directory)
    elif type(directory)==list:
        for folder in range(len(directory)):
            [out.append(file) for file in find_files(directory[folder])]
    return(
        sorted(out))
# 이미지 변수 생성
def img_feature(dirs):
    df=pd.DataFrame({'img_dirs':dirs})
    for i in tqdm(range(len(dirs))):
        img=cv2.imread(dirs[i])
        raw_img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC)
        contrast_img=img_Contrast(img, 3, tileGridSize=(4,3))

        # 시든 잎 추출
        extract_img=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255))
        df.loc[i,'del_leaf']=img_to_weight(extract_img,16900)

        # 청경채 추출
        extract_img=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출
        df.loc[i,'pred_leaf1']=img_to_weight(extract_img,16900)
        contrast_img=img_Contrast(img, 3, tileGridSize=(5,5))
        extract_img=img_extract(img,contrast_img,upper=(77,255,130))# 원본에서 추출
        df.loc[i,'pred_leaf2']=img_to_weight(extract_img,16900)
        df.loc[i,'pred_leaf_mean']=(df.loc[i,'pred_leaf1']+df.loc[i,'pred_leaf2'])/2

        #RGB 추출
        df.loc[i,["r_m","g_m","b_m","r_sd","g_sd","b_sd"]]=rgb_stat(extract_img)
    return df

자료생성¶

In [5]:

tr_directory=[f"{main_dir}/train/CASE{i:02d}/image" for i in range(1,76)]
tr_img_dirs=file_list(tr_directory)
tr_img_dirs.remove(f'{main_dir}/train/CASE45/image/CASE45_17.png')
te_img_dirs=file_list(f"{main_dir}/test/image")
label_dfs=list()
for file in tqdm([f"{main_dir}/train/CASE{i:02d}/label.csv" for i in range(1,76)]):
    temp_df=pd.read_csv(file)
    for i, img_file in enumerate(temp_df.img_name):
        time_df=pd.read_csv(
            f"{main_dir}/train/CASE{img_file[4:6]}/meta/{img_file.replace('jpg','png').replace('png','csv')}")
        time_df=time_df.sort_values('시간')
        time=time_df.loc[0,'시간']
        temp_df.loc[i,'date']=pd.to_datetime(time).date()
    label_dfs.append(temp_df)
label_df=pd.concat(label_dfs)

label_df['case']=[i[:6] for i in label_df['img_name']]
merge_df=label_df.copy()
merge_df.columns=['img_name','now_weight','date','case']
merge_df=merge_df.drop("img_name",axis=1)
merge_df['date']=merge_df.date+pd.to_timedelta(1,unit='day')
label_df=pd.merge(label_df,merge_df,how='left',on=['case','date'])
del merge_df

  0%|          | 0/75 [00:00<?, ?it/s]

In [6]:

if not('tr_df.csv' in os.listdir("/root/jupyter/데이콘/청경채/input/")):
    tr_df=img_feature(tr_img_dirs)
    te_df=img_feature(te_img_dirs)
    tr_df.to_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv',index=False)
    te_df.to_csv('/root/jupyter/데이콘/청경채/input/te_df.csv',index=False)
else:
    tr_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv')
    te_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/te_df.csv')
#라벨
for i in tqdm(range(tr_df.shape[0])):
    tr_df.loc[i,['leaf_weight','date','now_weight']]=label_df.loc[
        label_df.img_name==tr_df.img_dirs[i].split('/')[-1],['leaf_weight','date','now_weight']].values[0]

  0%|          | 0/1591 [00:00<?, ?it/s]

삭제할 자료¶

CASE 2_10, 2_11, 34_01, 40_01, 40_02, 44_01, 52_01, 56_01, 60_20~34, 63_01, 64_01 : 환경자료 결측

CASE 8, 9, 22, 23, 26, 30, 31, 49, 59, 71, 72, 73 : 환경자료 결측

CASE 35_01, 41_01, 44_02, 45_01, 52_02, 53_01, 56_02, 57_01, 63_02 : 부분결측(제거)

CASE 34, 35, 48 : EC 결측

CASE 32_15, 51_11 : Co2 이상

In [7]:

env_na_p=[f"CASE{i}" for i in set([f"60_{i}" for i in range(20,34+1)]).union(
    set(['02_10','02_11','34_01','40_01','40_02','44_01','52_01','56_01','63_01','64_01']))]
env_na_a=[f"CASE{i}" for i in ['08','09','22','23','26','30','31','49','59','71','72','73']]
partial_na=[f"CASE{i}" for i in ["35_01","41_01","44_02","45_01","52_02","53_01","57_01","63_02"]]
ec_na=[f"CASE{i}" for i in ["34","35","48"]]

tr_df['na_label']=False
for i in (env_na_p+env_na_a+partial_na+ec_na):
    tr_df.loc[tr_df['img_dirs'].str.contains(i),"na_label"]=True

In [8]:

for i, filename in tqdm(enumerate(tr_df.img_dirs)):
    temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv'))
    time_df=time_df.sort_values('시간')
    temp_df.시간=pd.to_datetime(temp_df.시간)
    temp_df['청색광추정광량']=(temp_df['총추정광량']-
                        temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()]

    aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5)
    night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5)
    co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계
    zero_ec_cnt=sum(temp_df['EC관측치']==0)
    disease_signal=co2_ratio*zero_ec_cnt

    aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5)
    night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5)
    night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5)
    m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True)
    m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True)    
    # 결측자료 처리
    if np.isnan(aftn_ec):
        if tr_df['del_leaf'][i]>10:
            aftn_ec=night_ec1*1.5
        else:
            aftn_ec=night_ec2
    if np.isnan(m_temp):
        m_temp=temp_df['내부온도관측치'].mean(skipna=True)
    if np.isnan(m_humidity):
        m_humidity=temp_df['내부습도관측치'].mean(skipna=True)

    if np.isnan(night_ec1):
        night_ec1=0
    if np.isnan(co2_ratio):
        if tr_df.pred_leaf_mean[i]>50:
            co2_ratio = 0.5
        else:
            co2_ratio = 1.5
    disease_signal=co2_ratio*zero_ec_cnt
    tr_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal',
                 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\
        co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity

0it [00:00, ?it/s]

In [9]:

for i, filename in tqdm(enumerate(te_df.img_dirs)):
    temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv'))
    time_df=time_df.sort_values('시간')
    temp_df.시간=pd.to_datetime(temp_df.시간)
    temp_df['청색광추정광량']=(temp_df['총추정광량']-
                        temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()]

    aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5)
    night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5)
    co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계
    zero_ec_cnt=sum(temp_df['EC관측치']==0)


    aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5)
    night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5)
    night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5)
    m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True)
    m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True)
    # 결측자료 처리
    if np.isnan(aftn_ec):
        if te_df['del_leaf'][i]>10:
            aftn_ec=night_ec1*1.5
        else:
            aftn_ec=night_ec2
    if np.isnan(m_temp):
        m_temp=temp_df['내부온도관측치'].mean(skipna=True)
    if np.isnan(m_humidity):
        m_humidity=temp_df['내부습도관측치'].mean(skipna=True)

    if np.isnan(night_ec1):
        night_ec1=0
    if np.isnan(co2_ratio):
        if te_df.pred_leaf_mean[i]>50:
            co2_ratio = 0.5
        else:
            co2_ratio = 1.5
    disease_signal=co2_ratio*zero_ec_cnt

    te_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal',
                 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\
        co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity

0it [00:00, ?it/s]

청경채 무게 추정¶

In [10]:

df=tr_df[['img_dirs','leaf_weight']].dropna().reset_index()
train,valid=train_test_split(df, test_size=0.33, random_state=42)

In [11]:

df[['img_dirs','leaf_weight']].isna().sum()

Out[11]:

img_dirs       0
leaf_weight    0
dtype: int64

In [23]:

def tr_gen():
    df=train[['img_dirs','leaf_weight']].dropna().reset_index()
    for i in range(df.shape[0]):
        img=cv2.imread(df.img_dirs[i])
        img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC)
        contrast_img=img_Contrast(img, 3, tileGridSize=(4,3))
        extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255))
        extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출
        extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1)
        yield (extract_img/255, df.leaf_weight[i])
def va_gen():
    df=valid[['img_dirs','leaf_weight']].dropna().reset_index()
    for i in range(df.shape[0]):
        img=cv2.imread(df.img_dirs[i])
        img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC)
        contrast_img=img_Contrast(img, 3, tileGridSize=(4,3))
        extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255))
        extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출
        extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1)
        yield (extract_img/255, df.leaf_weight[i])
def check_gen():
    df=tr_df[['img_dirs','leaf_weight']].dropna().reset_index()
    for i in range(df.shape[0]):
        img=cv2.imread(df.img_dirs[i])
        img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC)
        contrast_img=img_Contrast(img, 3, tileGridSize=(4,3))
        extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255))
        extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출
        extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1)
        yield (extract_img/255, df.leaf_weight[i])

def te_gen():
    df=te_df[['img_dirs']].reset_index()
    for i in range(df.shape[0]):
        img=cv2.imread(df.img_dirs[i])
        img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC)
        contrast_img=img_Contrast(img, 3, tileGridSize=(4,3))
        extract_img1=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255))
        extract_img2=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출
        extract_img=cv2.addWeighted(extract_img1,1,extract_img2,1,1)
        yield (extract_img/255, np.nan)
def NMAE(true, pred):
    mae = np.mean(np.abs(true-pred))
    score = mae / np.mean(np.abs(true))
    return score

def nmae_keras(y_true, y_pred):
    score = tf.py_function(func=NMAE, inp=[y_true, y_pred], Tout=tf.float32,  name='name')
    return score

In [24]:

tr_data=tf.data.Dataset.from_generator(tr_gen,(tf.float32,tf.float32))
tr_data=tr_data.cache().batch(12).prefetch(buffer_size=10)
va_data=tf.data.Dataset.from_generator(va_gen,(tf.float32,tf.float32))
va_data=va_data.cache().batch(12).prefetch(buffer_size=10)
te_data=tf.data.Dataset.from_generator(te_gen,(tf.float32,tf.float32))
te_data=te_data.cache().batch(12).prefetch(buffer_size=10)
ch_data=tf.data.Dataset.from_generator(check_gen,(tf.float32,tf.float32))
ch_data=ch_data.cache().batch(12).prefetch(buffer_size=10)

In [25]:

next(iter(va_data))[1]

Out[25]:

<tf.Tensor: shape=(12,), dtype=float32, numpy=
array([ 11.885, 157.091,   0.189,  54.305, 133.164,   2.36 , 131.028,
        20.628,  28.062,  48.271,   2.427,  12.335], dtype=float32)>

In [28]:

if not('forecast_weight_best_model_v3.h5' in os.listdir("/root/jupyter/데이콘/청경채/output/")):
    tf.random.set_seed(42)
    inp = tf.keras.Input(shape=(820, 616, 3),dtype=tf.float32)
    
    x = tf.keras.layers.Conv2D(32, 3, strides=2, padding="same")(inp)
    x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Activation("relu")(x)

    x = tf.keras.layers.Conv2D(64, 3, padding="same")(x)
    x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Activation("relu")(x)

    previous_block_activation = x
    
    for size in [128, 256, 512]:
        x = tf.keras.layers.Activation("relu")(x)
        x = tf.keras.layers.SeparableConv2D(size, 3, padding="same")(x)
        x = tf.keras.layers.BatchNormalization()(x)

        x = tf.keras.layers.Activation("relu")(x)
        x = tf.keras.layers.SeparableConv2D(size, 3, padding="same")(x)
        x = tf.keras.layers.BatchNormalization()(x)

        x = tf.keras.layers.MaxPooling2D(3, strides=2, padding="same")(x)

        # Project residual
        residual = tf.keras.layers.Conv2D(size, 1, strides=2, padding="same")(
            previous_block_activation
        )
        x = tf.keras.layers.add([x, residual])  # Add back residual
        previous_block_activation = x  # Set aside next residual

    x = tf.keras.layers.SeparableConv2D(1024, 3, padding="same")(x)

    x = tf.keras.layers.GlobalAveragePooling2D()(x)

    x = tf.keras.layers.Dropout(0.5)(x)
    out = tf.keras.layers.Dense(units=1)(x)
    
    model = tf.keras.Model(inp, out)
    early = tf.keras.callbacks.EarlyStopping(
        monitor='val_loss',mode="min", patience=10)
    lr_reduce=tf.keras.callbacks.ReduceLROnPlateau(
        monitor='val_loss',patience=3,verbose=1,min_delta=0.001)
    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01),
                  loss='mse',metrics=['mae'])
    model.fit(tr_data,verbose=1,callbacks =[early,lr_reduce],
              epochs=50,validation_data=va_data)

    # fine tuning
    for i in model.layers[0:-4]:
        i.trainable=False
    cp_callback = tf.keras.callbacks.ModelCheckpoint(
        filepath='/root/jupyter/데이콘/청경채/model3/now_weight_{val_loss:.2f}.h5',
        monitor='val_loss',mode='min',verbose=1)
    early = tf.keras.callbacks.EarlyStopping(monitor='val_loss',mode="min", patience=10)
    lr_reduce=tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss',patience=3,verbose=1)
    tf.random.set_seed(42)
    model.fit(tr_data,verbose=1,callbacks =[early,lr_reduce,cp_callback],
              epochs=300,validation_data=va_data)

    file_loss=min([i.split('/')[-1].split('_')[-1].replace('.h5','')
                   for i in os.listdir('/root/jupyter/데이콘/청경채/model3/')])
    model=tf.keras.models.load_model(f'/root/jupyter/데이콘/청경채/model3/now_weight_{file_loss}.h5')
    tf.keras.models.save_model(model,f'/root/jupyter/데이콘/청경채/output/forecast_weight_best_model_v3.h5')
else:
    model=tf.keras.models.load_model(f'/root/jupyter/데이콘/청경채/output/forecast_weight_best_model_v3.h5')

Epoch 1/50
89/89 [==============================] - 109s 1s/step - loss: 5688.1016 - mae: 44.0527 - val_loss: 15562.8027 - val_mae: 110.5253
Epoch 2/50
89/89 [==============================] - 106s 1s/step - loss: 2460.1113 - mae: 30.6779 - val_loss: 5831.2251 - val_mae: 68.4424
Epoch 3/50
89/89 [==============================] - 107s 1s/step - loss: 1793.7635 - mae: 25.7197 - val_loss: 8407.2588 - val_mae: 87.1451
Epoch 4/50
89/89 [==============================] - 107s 1s/step - loss: 1451.2693 - mae: 24.0861 - val_loss: 5336.9126 - val_mae: 55.4451
Epoch 5/50
89/89 [==============================] - 106s 1s/step - loss: 1386.1694 - mae: 23.2083 - val_loss: 402.9470 - val_mae: 11.4579
Epoch 6/50
89/89 [==============================] - 107s 1s/step - loss: 1429.0300 - mae: 23.5776 - val_loss: 2250.6030 - val_mae: 42.6930
Epoch 7/50
89/89 [==============================] - 107s 1s/step - loss: 1710.6003 - mae: 25.5833 - val_loss: 4405.1421 - val_mae: 54.3167
Epoch 8/50
89/89 [==============================] - 106s 1s/step - loss: 1697.0322 - mae: 26.0049 - val_loss: 749.8641 - val_mae: 21.0604

Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.0009999999776482583.
Epoch 9/50
89/89 [==============================] - 106s 1s/step - loss: 816.3369 - mae: 16.6580 - val_loss: 13658.1074 - val_mae: 114.6699
Epoch 10/50
89/89 [==============================] - 105s 1s/step - loss: 734.0446 - mae: 15.1714 - val_loss: 13992.6562 - val_mae: 115.2016
Epoch 11/50
89/89 [==============================] - 106s 1s/step - loss: 617.8029 - mae: 14.3338 - val_loss: 20528.9492 - val_mae: 139.4232

Epoch 00011: ReduceLROnPlateau reducing learning rate to 9.999999310821295e-05.
Epoch 12/50
89/89 [==============================] - 106s 1s/step - loss: 709.4712 - mae: 14.7965 - val_loss: 9054.4355 - val_mae: 91.8065
Epoch 13/50
89/89 [==============================] - 105s 1s/step - loss: 666.8051 - mae: 14.5698 - val_loss: 4217.9058 - val_mae: 62.4500
Epoch 14/50
89/89 [==============================] - 107s 1s/step - loss: 572.8399 - mae: 14.0790 - val_loss: 2282.8638 - val_mae: 45.6461

Epoch 00014: ReduceLROnPlateau reducing learning rate to 9.999999019782991e-06.
Epoch 15/50
89/89 [==============================] - 106s 1s/step - loss: 625.3889 - mae: 14.4341 - val_loss: 2141.7351 - val_mae: 44.1427
Epoch 1/300
89/89 [==============================] - 107s 1s/step - loss: 482.3590 - mae: 13.4125 - val_loss: 1940.7063 - val_mae: 41.9163

Epoch 00001: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1940.71.h5
Epoch 2/300
89/89 [==============================] - 105s 1s/step - loss: 490.9116 - mae: 13.2105 - val_loss: 1767.5862 - val_mae: 39.8894

Epoch 00002: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1767.59.h5
Epoch 3/300
89/89 [==============================] - 106s 1s/step - loss: 495.5956 - mae: 12.8869 - val_loss: 1768.0610 - val_mae: 39.8957

Epoch 00003: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1768.06.h5
Epoch 4/300
89/89 [==============================] - 106s 1s/step - loss: 537.9999 - mae: 13.3342 - val_loss: 1653.8525 - val_mae: 38.5116

Epoch 00004: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1653.85.h5
Epoch 5/300
89/89 [==============================] - 106s 1s/step - loss: 469.2600 - mae: 12.6919 - val_loss: 1582.7653 - val_mae: 37.6222

Epoch 00005: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1582.77.h5
Epoch 6/300
89/89 [==============================] - 105s 1s/step - loss: 500.2566 - mae: 13.3267 - val_loss: 1664.7568 - val_mae: 38.6509

Epoch 00006: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1664.76.h5
Epoch 7/300
89/89 [==============================] - 107s 1s/step - loss: 524.7254 - mae: 13.3676 - val_loss: 1572.4323 - val_mae: 37.4957

Epoch 00007: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1572.43.h5
Epoch 8/300
89/89 [==============================] - 106s 1s/step - loss: 486.0138 - mae: 12.9938 - val_loss: 1623.8711 - val_mae: 38.1453

Epoch 00008: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1623.87.h5
Epoch 9/300
89/89 [==============================] - 105s 1s/step - loss: 529.1871 - mae: 13.4911 - val_loss: 1757.0791 - val_mae: 39.7688

Epoch 00009: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1757.08.h5
Epoch 10/300
89/89 [==============================] - 104s 1s/step - loss: 610.0847 - mae: 13.8014 - val_loss: 1811.8817 - val_mae: 40.4208

Epoch 00010: ReduceLROnPlateau reducing learning rate to 9.99999883788405e-07.

Epoch 00010: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1811.88.h5
Epoch 11/300
89/89 [==============================] - 105s 1s/step - loss: 539.5002 - mae: 13.5189 - val_loss: 1759.1317 - val_mae: 39.7936

Epoch 00011: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1759.13.h5
Epoch 12/300
89/89 [==============================] - 105s 1s/step - loss: 685.1992 - mae: 14.2230 - val_loss: 1732.4094 - val_mae: 39.4723

Epoch 00012: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1732.41.h5
Epoch 13/300
89/89 [==============================] - 105s 1s/step - loss: 648.0275 - mae: 14.2835 - val_loss: 1725.6875 - val_mae: 39.3913

Epoch 00013: ReduceLROnPlateau reducing learning rate to 9.99999883788405e-08.

Epoch 00013: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1725.69.h5
Epoch 14/300
89/89 [==============================] - 106s 1s/step - loss: 561.6453 - mae: 13.8804 - val_loss: 1720.8232 - val_mae: 39.3325

Epoch 00014: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1720.82.h5
Epoch 15/300
89/89 [==============================] - 105s 1s/step - loss: 613.6439 - mae: 14.1456 - val_loss: 1719.5625 - val_mae: 39.3173

Epoch 00015: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1719.56.h5
Epoch 16/300
89/89 [==============================] - 105s 1s/step - loss: 654.1337 - mae: 14.1808 - val_loss: 1719.8595 - val_mae: 39.3209

Epoch 00016: ReduceLROnPlateau reducing learning rate to 9.999998695775504e-09.

Epoch 00016: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1719.86.h5
Epoch 17/300
89/89 [==============================] - 104s 1s/step - loss: 604.9459 - mae: 13.9551 - val_loss: 1718.9264 - val_mae: 39.3096

Epoch 00017: saving model to /root/jupyter/데이콘/청경채/model3/now_weight_1718.93.h5

In [31]:

tr_df.loc[~tr_df.leaf_weight.isna(),'cnn_now_weight']=model.predict(ch_data)
te_df['cnn_now_weight']=model.predict(te_data)

In [32]:

plt.scatter(tr_df.cnn_now_weight,tr_df.now_weight)

Out[32]:

<matplotlib.collections.PathCollection at 0x7f32181fb7f0>

In [33]:

plt.scatter(tr_df.pred_leaf_mean,tr_df.leaf_weight)

Out[33]:

<matplotlib.collections.PathCollection at 0x7f321807eac8>

In [34]:

submit=pd.read_csv(f"{main_dir}/sample_submission.csv")
submit['leaf_weight']=te_df['cnn_now_weight']
submit.to_csv(f"/root/jupyter/데이콘/청경채/output/submit_8.csv",index=False)

In [ ]: