Add learning_rate_decay
This commit is contained in:
parent
70eb2b508f
commit
c89fd7249a
3 changed files with 16 additions and 14 deletions
|
@ -7,6 +7,11 @@ local function minibatch_adam(model, criterion, eval_metric,
|
||||||
config)
|
config)
|
||||||
local parameters, gradParameters = model:getParameters()
|
local parameters, gradParameters = model:getParameters()
|
||||||
config = config or {}
|
config = config or {}
|
||||||
|
if config.xEvalCount == nil then
|
||||||
|
config.xEvalCount = 0
|
||||||
|
config.learningRate = config.xLearningRate
|
||||||
|
end
|
||||||
|
|
||||||
local sum_loss = 0
|
local sum_loss = 0
|
||||||
local sum_eval = 0
|
local sum_eval = 0
|
||||||
local count_loss = 0
|
local count_loss = 0
|
||||||
|
@ -52,11 +57,14 @@ local function minibatch_adam(model, criterion, eval_metric,
|
||||||
return f, gradParameters
|
return f, gradParameters
|
||||||
end
|
end
|
||||||
optim.adam(feval, parameters, config)
|
optim.adam(feval, parameters, config)
|
||||||
|
config.xEvalCount = config.xEvalCount + batch_size
|
||||||
|
config.learningRate = config.xLearningRate / (1 + config.xEvalCount * config.xLearningRateDecay)
|
||||||
c = c + 1
|
c = c + 1
|
||||||
if c % 50 == 0 then
|
if c % 50 == 0 then
|
||||||
collectgarbage()
|
collectgarbage()
|
||||||
xlua.progress(t, train_x:size(1))
|
xlua.progress(t, train_x:size(1))
|
||||||
end
|
end
|
||||||
|
|
||||||
end
|
end
|
||||||
xlua.progress(train_x:size(1), train_x:size(1))
|
xlua.progress(train_x:size(1), train_x:size(1))
|
||||||
return { loss = sum_loss / count_loss, MSE = sum_eval / count_loss, PSNR = 10 * math.log10(1 / (sum_eval / count_loss))}, instance_loss
|
return { loss = sum_loss / count_loss, MSE = sum_eval / count_loss, PSNR = 10 * math.log10(1 / (sum_eval / count_loss))}, instance_loss
|
||||||
|
|
|
@ -58,6 +58,7 @@ cmd:option("-resize_blur_min", 0.85, 'min blur parameter for ResizeImage')
|
||||||
cmd:option("-resize_blur_max", 1.05, 'max blur parameter for ResizeImage')
|
cmd:option("-resize_blur_max", 1.05, 'max blur parameter for ResizeImage')
|
||||||
cmd:option("-oracle_rate", 0.0, '')
|
cmd:option("-oracle_rate", 0.0, '')
|
||||||
cmd:option("-oracle_drop_rate", 0.5, '')
|
cmd:option("-oracle_drop_rate", 0.5, '')
|
||||||
|
cmd:option("-learning_rate_decay", 3.0e-7, 'learning rate decay (learning_rate * 1/(1+num_of_data*patches*epoch))')
|
||||||
|
|
||||||
local function to_bool(settings, name)
|
local function to_bool(settings, name)
|
||||||
if settings[name] == 1 then
|
if settings[name] == 1 then
|
||||||
|
|
21
train.lua
21
train.lua
|
@ -100,7 +100,6 @@ local function create_criterion(model)
|
||||||
local offset = reconstruct.offset_size(model)
|
local offset = reconstruct.offset_size(model)
|
||||||
local output_w = settings.crop_size - offset * 2
|
local output_w = settings.crop_size - offset * 2
|
||||||
local weight = torch.Tensor(3, output_w * output_w)
|
local weight = torch.Tensor(3, output_w * output_w)
|
||||||
|
|
||||||
weight[1]:fill(0.29891 * 3) -- R
|
weight[1]:fill(0.29891 * 3) -- R
|
||||||
weight[2]:fill(0.58661 * 3) -- G
|
weight[2]:fill(0.58661 * 3) -- G
|
||||||
weight[3]:fill(0.11448 * 3) -- B
|
weight[3]:fill(0.11448 * 3) -- B
|
||||||
|
@ -223,8 +222,8 @@ local function remove_small_image(x)
|
||||||
local new_x = {}
|
local new_x = {}
|
||||||
for i = 1, #x do
|
for i = 1, #x do
|
||||||
local x_s = compression.size(x[i])
|
local x_s = compression.size(x[i])
|
||||||
if x_s[2] / settings.scale > settings.crop_size + 16 and
|
if x_s[2] / settings.scale > settings.crop_size + 32 and
|
||||||
x_s[3] / settings.scale > settings.crop_size + 16 then
|
x_s[3] / settings.scale > settings.crop_size + 32 then
|
||||||
table.insert(new_x, x[i])
|
table.insert(new_x, x[i])
|
||||||
end
|
end
|
||||||
if i % 100 == 0 then
|
if i % 100 == 0 then
|
||||||
|
@ -253,10 +252,10 @@ local function train()
|
||||||
local x = remove_small_image(torch.load(settings.images))
|
local x = remove_small_image(torch.load(settings.images))
|
||||||
local train_x, valid_x = split_data(x, math.max(math.floor(settings.validation_rate * #x), 1))
|
local train_x, valid_x = split_data(x, math.max(math.floor(settings.validation_rate * #x), 1))
|
||||||
local adam_config = {
|
local adam_config = {
|
||||||
learningRate = settings.learning_rate,
|
xLearningRate = settings.learning_rate,
|
||||||
xBatchSize = settings.batch_size,
|
xBatchSize = settings.batch_size,
|
||||||
|
xLearningRateDecay = settings.learning_rate_decay
|
||||||
}
|
}
|
||||||
local lrd_count = 0
|
|
||||||
local ch = nil
|
local ch = nil
|
||||||
if settings.color == "y" then
|
if settings.color == "y" then
|
||||||
ch = 1
|
ch = 1
|
||||||
|
@ -285,10 +284,12 @@ local function train()
|
||||||
ch, settings.crop_size, settings.crop_size)
|
ch, settings.crop_size, settings.crop_size)
|
||||||
end
|
end
|
||||||
local instance_loss = nil
|
local instance_loss = nil
|
||||||
|
|
||||||
for epoch = 1, settings.epoch do
|
for epoch = 1, settings.epoch do
|
||||||
model:training()
|
model:training()
|
||||||
print("# " .. epoch)
|
print("# " .. epoch)
|
||||||
|
if adam_config.learningRate then
|
||||||
|
print("learning rate: " .. adam_config.learningRate)
|
||||||
|
end
|
||||||
print("## resampling")
|
print("## resampling")
|
||||||
if instance_loss then
|
if instance_loss then
|
||||||
-- active learning
|
-- active learning
|
||||||
|
@ -323,7 +324,6 @@ local function train()
|
||||||
end
|
end
|
||||||
if score.loss < best_score then
|
if score.loss < best_score then
|
||||||
local test_image = image_loader.load_float(settings.test) -- reload
|
local test_image = image_loader.load_float(settings.test) -- reload
|
||||||
lrd_count = 0
|
|
||||||
best_score = score.loss
|
best_score = score.loss
|
||||||
print("* update best model")
|
print("* update best model")
|
||||||
if settings.save_history then
|
if settings.save_history then
|
||||||
|
@ -351,13 +351,6 @@ local function train()
|
||||||
save_test_scale(model, test_image, log)
|
save_test_scale(model, test_image, log)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
else
|
|
||||||
lrd_count = lrd_count + 1
|
|
||||||
if lrd_count > 2 then
|
|
||||||
adam_config.learningRate = adam_config.learningRate * 0.874
|
|
||||||
print("* learning rate decay: " .. adam_config.learningRate)
|
|
||||||
lrd_count = 0
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
print("PSNR: " .. score.PSNR .. ", loss: " .. score.loss .. ", Minimum loss: " .. best_score)
|
print("PSNR: " .. score.PSNR .. ", loss: " .. score.loss .. ", Minimum loss: " .. best_score)
|
||||||
collectgarbage()
|
collectgarbage()
|
||||||
|
|
Loading…
Reference in a new issue