From 5548e1c998f71402ad312037fb0cc02711456ff9 Mon Sep 17 00:00:00 2001 From: curious21cn Date: Mon, 14 Dec 2015 16:22:45 -0500 Subject: [PATCH 1/6] implemented rls_insta INSTA for elastic net --- gurls/optimizers/rls_insta.m | 69 ++++++++++++++++++++++++++++++++++ gurls/utils/rls_insta_driver.m | 58 ++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 gurls/optimizers/rls_insta.m create mode 100644 gurls/utils/rls_insta_driver.m diff --git a/gurls/optimizers/rls_insta.m b/gurls/optimizers/rls_insta.m new file mode 100644 index 0000000..aee6c36 --- /dev/null +++ b/gurls/optimizers/rls_insta.m @@ -0,0 +1,69 @@ +function [cfr] = rls_insta (X, y, opt) + +% rls_insta(X,y,opt) +% computes a classifier for elastic nerwork using ISTA. +% The regularization parameter is set to the one found in opt.paramsel. +% In case of multiclass problems, the regularizers need to be combined with the opt.singlelambda function. +% +% INPUTS: +% -OPT: struct of options with the following fields: +% fields that need to be set through previous gurls tasks: +% - paramsel.lambdas (set by the paramsel_* routines) +% fields with default values set through the defopt function: +% - singlelambda +% +% For more information on standard OPT fields +% see also defopt +% +% OUTPUT: struct with the following fields: +% -W: matrix of coefficient vectors of rls estimator for each class +% -C: matrix of coefficient vectors of rls estimator for each class +% -X: empty matrix + +lambda = opt.singlelambda(opt.paramsel.lambdas); + +n = size(y,1); + +% load in parameters alpha +if isfield(opt.paramsel, 'insta_alpha') + insta_alpha=opt.paramsel.insta_alpha; + if insta_alpha <= 0 || insta_alpha > 1 + error('Invalid alpha'); + end +else + if opt.verbose + warning('alpha not defined. Use default value alpha=1 for LASSO'); + insta_alpha=1; + end +end + +% load in number of iterations or relative tolerance +Niter=-1; +relthre=1e-4; +if isfield(opt.paramsel, 'niter') + Niter=opt.paramsel.niter; +end +if isfield(opt.paramsel, 'relthre') + relthre=opt.paramsel.relthre; +end + +% check if matrices XtX and Xty have been previously computed during +% parameter selection +if isfield(opt.paramsel,'XtX'); + XtX = opt.paramsel.XtX; +else + XtX = X'*X; % d x d matrix. +end +if isfield(opt.paramsel,'XtX'); + Xty = opt.paramsel.Xty; +else + Xty = X'*y; % d x T matrix. +end + +% Determine Stop rule + +cfr.W = rls_insta_driver( XtX, Xty, n, lambda,insta_alpha,Niter,relthre,opt); +cfr.IndexFlag = ~~(cfr.W); +cfr.C = []; +cfr.X = []; + diff --git a/gurls/utils/rls_insta_driver.m b/gurls/utils/rls_insta_driver.m new file mode 100644 index 0000000..f025e5a --- /dev/null +++ b/gurls/utils/rls_insta_driver.m @@ -0,0 +1,58 @@ +function w=rls_insta_driver( XtX, Xty, n, lambda,inst_alpha,Niter,relthre,opt) +% rls_insta_driver( XtX, Xty, n, lambda,alpha,Niter,reltol,opt) +% Utility function used by rls_insta +% +% INPUTS: +% -XtX: symmetric dxd square matrix +% -Xty: dxT matrix +% -n: number of training samples +% -lambda: regularization parameter +% -alpha: l1 norm l2 norm weight parameter +% -Niter: maximum number of iteration +% -retol: relative tolernce for convergence +% -opt: use opt.verbose only +% +% OUTPUTS: +% -W: matrix of coefficient vector for linear RLS classifier + + gamma = n/(2*eigs(XtX,1)+lambda*(1-inst_alpha)); %Step size + + % make sure Niter, retol are proper + if ~isinteger(Niter) + if opt.verbose + warning('Interation number must be integer. Rounding opt.paramsel.niter'); + end + Niter = ceil(Niter); + end + if Niter<=0 && relthre<0 + if opt.verbose + warning(['Unvalid stopping rule for INSTA.',... + 'Using default relative tolerance = 1e-4']); + end + Niter = -1; + relthre=1e-4; + end + + % start insta + + w2=XtX\Xty; + w2(abs(w2)relthre*max([mean(abs(w1)),mean(abs(w2))]) + w1=w2; + w2=(1-lambda*gamma*(1-inst_alpha))*w2+2*gamma*(Xty-XtX*w2)/n; + w2(abs(w2) Date: Mon, 14 Dec 2015 16:45:13 -0500 Subject: [PATCH 2/6] Implemented INSTA --- gurls/optimizers/rls_insta.m | 14 +++++++++++--- gurls/utils/rls_insta_driver.m | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/gurls/optimizers/rls_insta.m b/gurls/optimizers/rls_insta.m index aee6c36..f5cdf2c 100644 --- a/gurls/optimizers/rls_insta.m +++ b/gurls/optimizers/rls_insta.m @@ -11,7 +11,11 @@ % - paramsel.lambdas (set by the paramsel_* routines) % fields with default values set through the defopt function: % - singlelambda -% +% fields that is optional +% - paramsel.insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) +% - paramsel.niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) +% - paramsel.relthre (relative convergence threshold for iteration to stop) +% % For more information on standard OPT fields % see also defopt % @@ -60,10 +64,14 @@ Xty = X'*y; % d x T matrix. end -% Determine Stop rule +% redo OLR based on non-sparsy components +% w = rls_insta_driver( XtX, Xty, n, lambda,insta_alpha,Niter,relthre,opt); +% cfr.IndexFlag = ~~(w); +% Xs=X(:,~~w); +% cfr.W=zeros(size(w)); +% cfr.W(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); cfr.W = rls_insta_driver( XtX, Xty, n, lambda,insta_alpha,Niter,relthre,opt); -cfr.IndexFlag = ~~(cfr.W); cfr.C = []; cfr.X = []; diff --git a/gurls/utils/rls_insta_driver.m b/gurls/utils/rls_insta_driver.m index f025e5a..ab21cb3 100644 --- a/gurls/utils/rls_insta_driver.m +++ b/gurls/utils/rls_insta_driver.m @@ -35,7 +35,7 @@ % start insta - w2=XtX\Xty; + w2=rls_primal_driver(XtX, Xty, n, 0); w2(abs(w2) Date: Mon, 14 Dec 2015 23:17:59 -0500 Subject: [PATCH 3/6] implemented INSTA --- gurls/gurls_defopt.m | 4 ++++ gurls/optimizers/rls_insta.m | 40 +++++++++++++++++----------------- gurls/utils/rls_insta_driver.m | 8 +++---- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/gurls/gurls_defopt.m b/gurls/gurls_defopt.m index 97c80d5..96c8ab4 100644 --- a/gurls/gurls_defopt.m +++ b/gurls/gurls_defopt.m @@ -76,4 +76,8 @@ opt.newprop( 'jobid', 1); opt.newprop( 'seq', {}); opt.newprop( 'process', {}); + %% INSTA options + opt.newprop( 'INSTAalpha',1); + opt.newprop( 'INSTAniter',inf); + opt.newprop( 'INSTArelthre',1e-4); end diff --git a/gurls/optimizers/rls_insta.m b/gurls/optimizers/rls_insta.m index f5cdf2c..df98363 100644 --- a/gurls/optimizers/rls_insta.m +++ b/gurls/optimizers/rls_insta.m @@ -12,9 +12,9 @@ % fields with default values set through the defopt function: % - singlelambda % fields that is optional -% - paramsel.insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) -% - paramsel.niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) -% - paramsel.relthre (relative convergence threshold for iteration to stop) +% - insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) +% - niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) +% - relthre (relative convergence threshold for iteration to stop) % % For more information on standard OPT fields % see also defopt @@ -29,26 +29,26 @@ n = size(y,1); % load in parameters alpha -if isfield(opt.paramsel, 'insta_alpha') - insta_alpha=opt.paramsel.insta_alpha; - if insta_alpha <= 0 || insta_alpha > 1 +if isprop(opt,'INSTAalpha') + INSTAalpha=opt.INSTAalpha; + if INSTAalpha <= 0 || INSTAalpha > 1 error('Invalid alpha'); end else if opt.verbose - warning('alpha not defined. Use default value alpha=1 for LASSO'); - insta_alpha=1; + fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); + INSTAalpha=1; end end % load in number of iterations or relative tolerance -Niter=-1; +Niter=inf; relthre=1e-4; -if isfield(opt.paramsel, 'niter') - Niter=opt.paramsel.niter; +if isprop(opt, 'INSTAniter') + Niter=opt.INSTAniter; end -if isfield(opt.paramsel, 'relthre') - relthre=opt.paramsel.relthre; +if isprop(opt, 'INSTArelthre') + relthre=opt.INSTArelthre; end % check if matrices XtX and Xty have been previously computed during @@ -65,13 +65,13 @@ end -% redo OLR based on non-sparsy components -% w = rls_insta_driver( XtX, Xty, n, lambda,insta_alpha,Niter,relthre,opt); -% cfr.IndexFlag = ~~(w); -% Xs=X(:,~~w); -% cfr.W=zeros(size(w)); -% cfr.W(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); -cfr.W = rls_insta_driver( XtX, Xty, n, lambda,insta_alpha,Niter,relthre,opt); +%redo OLR based on non-sparsy components +w = rls_insta_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt); +cfr.IndexFlag = ~~(w); +Xs=X(:,~~w); +cfr.Wr=zeros(size(w)); +cfr.Wr(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); +cfr.W = w; cfr.C = []; cfr.X = []; diff --git a/gurls/utils/rls_insta_driver.m b/gurls/utils/rls_insta_driver.m index ab21cb3..6d9156c 100644 --- a/gurls/utils/rls_insta_driver.m +++ b/gurls/utils/rls_insta_driver.m @@ -18,16 +18,16 @@ gamma = n/(2*eigs(XtX,1)+lambda*(1-inst_alpha)); %Step size % make sure Niter, retol are proper - if ~isinteger(Niter) + if (Niter-round(Niter))~=0 && ~isinf(Niter) if opt.verbose - warning('Interation number must be integer. Rounding opt.paramsel.niter'); + fprintf('\t...Interation number must be integer. Rounding opt.paramsel.niter\n'); end Niter = ceil(Niter); end if Niter<=0 && relthre<0 if opt.verbose - warning(['Unvalid stopping rule for INSTA.',... - 'Using default relative tolerance = 1e-4']); + fprintf(['\t...Unvalid stopping rule for INSTA.',... + 'Using default relative tolerance = 1e-4\n']); end Niter = -1; relthre=1e-4; From 525c9302e86660efd6f05171ef8212533a0e4e46 Mon Sep 17 00:00:00 2001 From: curious21cn Date: Tue, 15 Dec 2015 15:17:17 -0500 Subject: [PATCH 4/6] Implemented parameter selection for ISTA --- gurls/gurls_defopt.m | 4 +- gurls/optimizers/{rls_insta.m => rls_ista.m} | 13 +- gurls/paramsel/paramsel_hoista.m | 134 ++++++++++++++++++ gurls/quickanddirty/ISTA_test.m | 10 ++ .../{rls_insta_driver.m => rls_ista_driver.m} | 14 +- 5 files changed, 161 insertions(+), 14 deletions(-) rename gurls/optimizers/{rls_insta.m => rls_ista.m} (87%) create mode 100644 gurls/paramsel/paramsel_hoista.m create mode 100644 gurls/quickanddirty/ISTA_test.m rename gurls/utils/{rls_insta_driver.m => rls_ista_driver.m} (84%) diff --git a/gurls/gurls_defopt.m b/gurls/gurls_defopt.m index 96c8ab4..c614cde 100644 --- a/gurls/gurls_defopt.m +++ b/gurls/gurls_defopt.m @@ -76,8 +76,10 @@ opt.newprop( 'jobid', 1); opt.newprop( 'seq', {}); opt.newprop( 'process', {}); - %% INSTA options + %% ISTA options opt.newprop( 'INSTAalpha',1); opt.newprop( 'INSTAniter',inf); opt.newprop( 'INSTArelthre',1e-4); + opt.newprop( 'INSTASparsitylvl',inf); + opt.newprop( 'INSTASAccuReq',0); end diff --git a/gurls/optimizers/rls_insta.m b/gurls/optimizers/rls_ista.m similarity index 87% rename from gurls/optimizers/rls_insta.m rename to gurls/optimizers/rls_ista.m index df98363..4702de4 100644 --- a/gurls/optimizers/rls_insta.m +++ b/gurls/optimizers/rls_ista.m @@ -1,4 +1,4 @@ -function [cfr] = rls_insta (X, y, opt) +function [cfr] = rls_ista (X, y, opt) % rls_insta(X,y,opt) % computes a classifier for elastic nerwork using ISTA. @@ -11,7 +11,7 @@ % - paramsel.lambdas (set by the paramsel_* routines) % fields with default values set through the defopt function: % - singlelambda -% fields that is optional +% fields with default values set through the defopt function: % - insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) % - niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) % - relthre (relative convergence threshold for iteration to stop) @@ -66,12 +66,13 @@ %redo OLR based on non-sparsy components -w = rls_insta_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt); +w = rls_ista_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt.verbose); cfr.IndexFlag = ~~(w); -Xs=X(:,~~w); -cfr.Wr=zeros(size(w)); -cfr.Wr(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); +% Xs=X(:,~~w); +% cfr.Wr=zeros(size(w)); +% cfr.Wr(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); cfr.W = w; cfr.C = []; cfr.X = []; +plot(w) diff --git a/gurls/paramsel/paramsel_hoista.m b/gurls/paramsel/paramsel_hoista.m new file mode 100644 index 0000000..ae1a412 --- /dev/null +++ b/gurls/paramsel/paramsel_hoista.m @@ -0,0 +1,134 @@ +function vout = paramsel_hoista(X, y, opt) +% paramsel_hoinsta(X,Y,OPT) +% Performs parameter selection INSTA for elastic net is used. +% The hold-out approach is used. +% The performance measure specified by opt.hoperf is maximized. +% +% INPUTS: +% -OPT: struct of options with the following fields: +% fields that need to be set through previous gurls tasks: +% - split (set by the split_* routine) +% fields with default values set through the defopt function: +% - nlambda +% - hoperf +% - nholdouts +% +% For more information on standard OPT fields +% see also defopt +% +% OUTPUTS: structure with the following fields: +% -lambdas_round: cell array (opt.nholdoutsX1). For each split a cell contains the +% values of the regularization parameter lambda minimizing the +% validation error for each class. +% -perf: cell array (opt.nholdouts). For each split a cell contains a matrix +% with the validation error for each lambda guess and for each class +% -guesses: cell array (opt.nholdoutsX1). For each split a cell contains an +% array of guesses for the regularization parameter lambda +% -lambdas: mean of the optimal lambdas across splits + +if isprop(opt,'paramsel') + vout = opt.paramsel; % lets not overwrite existing parameters. + % unless they have the same name + + if isfield(opt.paramsel,'perf') + vout = rmfield(vout,'perf'); + end + if isfield(opt.paramsel,'guesses') + vout = rmfield(vout,'guesses'); + end +else + opt.newprop('paramsel', struct()); +end + +% load in number of iterations or relative tolerance +Niter=inf; +relthre=1e-4; +if isprop(opt, 'INSTAniter') + Niter=opt.INSTAniter; +end +if isprop(opt, 'INSTArelthre') + relthre=opt.INSTArelthre; +end +% load in alpha for elastic net +if isprop(opt,'INSTAalpha') + INSTAalpha=opt.INSTAalpha; + if INSTAalpha <= 0 || INSTAalpha > 1 + error('Invalid alpha'); + end +else + if opt.verbose + fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); + INSTAalpha=1; + end +end + + +Xtytot = X'*y; + +d = size(X,2); +T = size(y,2); + +% Verify the parameter selection constraint is defined proper +opt.INSTASparsitylvl=ceil(opt.INSTASparsitylvl); +if (opt.INSTASAccuReq>0&&opt.INSTASparsitylvlopt.INSTASparsitylvl),:)=-1; + [~, idx] = max(apt,[],1); + else + Sparsity(min(ap,[],2) 1 + lambdas = cell2mat(vout.lambdas_round'); + vout.lambdas = mean(lambdas); +else + vout.lambdas = vout.lambdas_round{1}; +end diff --git a/gurls/quickanddirty/ISTA_test.m b/gurls/quickanddirty/ISTA_test.m new file mode 100644 index 0000000..51b90f3 --- /dev/null +++ b/gurls/quickanddirty/ISTA_test.m @@ -0,0 +1,10 @@ +% test ISTA on elastic net +name='ExampleExperiment'; +opt = gurls_defopt(name); +opt.seq={'split:ho','paramsel:hoista','rls:ista','pred:primal','perf:macroavg'}; +opt.process{1}=[2,2,2,0,0]; +opt.process{2}=[3,3,3,2,2]; +%opt.INSTASparsitylvl=1; +opt.INSTASAccuReq = 0.3; +gurls(Xtr,ytr,opt,1); +gurls(Xte,yte,opt,2); \ No newline at end of file diff --git a/gurls/utils/rls_insta_driver.m b/gurls/utils/rls_ista_driver.m similarity index 84% rename from gurls/utils/rls_insta_driver.m rename to gurls/utils/rls_ista_driver.m index 6d9156c..961e2e1 100644 --- a/gurls/utils/rls_insta_driver.m +++ b/gurls/utils/rls_ista_driver.m @@ -1,5 +1,5 @@ -function w=rls_insta_driver( XtX, Xty, n, lambda,inst_alpha,Niter,relthre,opt) -% rls_insta_driver( XtX, Xty, n, lambda,alpha,Niter,reltol,opt) +function w=rls_ista_driver( XtX, Xty, n, lambda,inst_alpha,Niter,relthre,verbose) +% rls_insta_driver( XtX, Xty, n, lambda,alpha,Niter,reltol,verbose) % Utility function used by rls_insta % % INPUTS: @@ -10,7 +10,7 @@ % -alpha: l1 norm l2 norm weight parameter % -Niter: maximum number of iteration % -retol: relative tolernce for convergence -% -opt: use opt.verbose only +% -verbose: output option % % OUTPUTS: % -W: matrix of coefficient vector for linear RLS classifier @@ -19,13 +19,13 @@ % make sure Niter, retol are proper if (Niter-round(Niter))~=0 && ~isinf(Niter) - if opt.verbose + if verbose fprintf('\t...Interation number must be integer. Rounding opt.paramsel.niter\n'); end Niter = ceil(Niter); end if Niter<=0 && relthre<0 - if opt.verbose + if verbose fprintf(['\t...Unvalid stopping rule for INSTA.',... 'Using default relative tolerance = 1e-4\n']); end @@ -33,7 +33,7 @@ relthre=1e-4; end - % start insta + % start ista w2=rls_primal_driver(XtX, Xty, n, 0); w2(abs(w2) Date: Tue, 15 Dec 2015 23:13:18 -0500 Subject: [PATCH 5/6] Implemented FISTA --- gurls/optimizers/rls_fista.m | 81 +++++++++++++++++++++++++++++++++ gurls/optimizers/rls_ista.m | 5 +- gurls/quickanddirty/ISTA_test.m | 10 ---- gurls/utils/rls_fista_driver.m | 62 +++++++++++++++++++++++++ gurls/utils/rls_ista_driver.m | 4 +- 5 files changed, 149 insertions(+), 13 deletions(-) create mode 100644 gurls/optimizers/rls_fista.m delete mode 100644 gurls/quickanddirty/ISTA_test.m create mode 100644 gurls/utils/rls_fista_driver.m diff --git a/gurls/optimizers/rls_fista.m b/gurls/optimizers/rls_fista.m new file mode 100644 index 0000000..1d4a306 --- /dev/null +++ b/gurls/optimizers/rls_fista.m @@ -0,0 +1,81 @@ +function [cfr] = rls_fista (X, y, opt) + +% rls_insta(X,y,opt) +% computes a classifier for elastic nerwork using ISTA. +% The regularization parameter is set to the one found in opt.paramsel. +% In case of multiclass problems, the regularizers need to be combined with the opt.singlelambda function. +% +% INPUTS: +% -OPT: struct of options with the following fields: +% fields that need to be set through previous gurls tasks: +% - paramsel.lambdas (set by the paramsel_* routines) +% fields with default values set through the defopt function: +% - singlelambda +% fields with default values set through the defopt function: +% - insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) +% - niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) +% - relthre (relative convergence threshold for iteration to stop) +% +% For more information on standard OPT fields +% see also defopt +% +% OUTPUT: struct with the following fields: +% -W: matrix of coefficient vectors of rls estimator for each class +% -C: matrix of coefficient vectors of rls estimator for each class +% -X: empty matrix + +lambda = opt.singlelambda(opt.paramsel.lambdas); + +n = size(y,1); + +% load in parameters alpha +if isprop(opt,'INSTAalpha') + INSTAalpha=opt.INSTAalpha; + if INSTAalpha <= 0 || INSTAalpha > 1 + error('Invalid alpha'); + end +else + if opt.verbose + fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); + INSTAalpha=1; + end +end + +% load in number of iterations or relative tolerance +Niter=inf; +relthre=1e-4; +if isprop(opt, 'INSTAniter') + Niter=opt.INSTAniter; +end +if isprop(opt, 'INSTArelthre') + relthre=opt.INSTArelthre; +end + +% check if matrices XtX and Xty have been previously computed during +% parameter selection +if isfield(opt.paramsel,'XtX'); + XtX = opt.paramsel.XtX; +else + XtX = X'*X; % d x d matrix. +end +if isfield(opt.paramsel,'XtX'); + Xty = opt.paramsel.Xty; +else + Xty = X'*y; % d x T matrix. +end + + +%redo OLR based on non-sparsy components + + +w = rls_fista_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt.verbose); + +cfr.IndexFlag = ~~(w); +% Xs=X(:,~~w); +% cfr.Wr=zeros(size(w)); +% cfr.Wr(~~w) = rls_primal_driver(Xs'*Xs,Xs'*y,n,0); +cfr.W = w; +cfr.C = []; +cfr.X = []; +%plot(w) + diff --git a/gurls/optimizers/rls_ista.m b/gurls/optimizers/rls_ista.m index 4702de4..51c9452 100644 --- a/gurls/optimizers/rls_ista.m +++ b/gurls/optimizers/rls_ista.m @@ -64,9 +64,10 @@ Xty = X'*y; % d x T matrix. end - %redo OLR based on non-sparsy components + w = rls_ista_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt.verbose); + cfr.IndexFlag = ~~(w); % Xs=X(:,~~w); % cfr.Wr=zeros(size(w)); @@ -74,5 +75,5 @@ cfr.W = w; cfr.C = []; cfr.X = []; -plot(w) +%plot(w) diff --git a/gurls/quickanddirty/ISTA_test.m b/gurls/quickanddirty/ISTA_test.m deleted file mode 100644 index 51b90f3..0000000 --- a/gurls/quickanddirty/ISTA_test.m +++ /dev/null @@ -1,10 +0,0 @@ -% test ISTA on elastic net -name='ExampleExperiment'; -opt = gurls_defopt(name); -opt.seq={'split:ho','paramsel:hoista','rls:ista','pred:primal','perf:macroavg'}; -opt.process{1}=[2,2,2,0,0]; -opt.process{2}=[3,3,3,2,2]; -%opt.INSTASparsitylvl=1; -opt.INSTASAccuReq = 0.3; -gurls(Xtr,ytr,opt,1); -gurls(Xte,yte,opt,2); \ No newline at end of file diff --git a/gurls/utils/rls_fista_driver.m b/gurls/utils/rls_fista_driver.m new file mode 100644 index 0000000..14e6c35 --- /dev/null +++ b/gurls/utils/rls_fista_driver.m @@ -0,0 +1,62 @@ +function w=rls_fista_driver( XtX, Xty, n, lambda,inst_alpha,Niter,relthre,verbose) +% rls_insta_driver( XtX, Xty, n, lambda,alpha,Niter,reltol,verbose) +% Utility function used by rls_insta +% +% INPUTS: +% -XtX: symmetric dxd square matrix +% -Xty: dxT matrix +% -n: number of training samples +% -lambda: regularization parameter +% -alpha: l1 norm l2 norm weight parameter +% -Niter: maximum number of iteration +% -retol: relative tolernce for convergence +% -verbose: output option +% +% OUTPUTS: +% -W: matrix of coefficient vector for linear RLS classifier + + gamma = n/(2*eigs(XtX,1)+lambda*(1-inst_alpha)); %Step size + + % make sure Niter, retol are proper + if (Niter-round(Niter))~=0 && ~isinf(Niter) + if verbose + fprintf('\t...Interation number must be integer. Rounding opt.paramsel.niter\n'); + end + Niter = ceil(Niter); + end + if Niter<=0 && relthre<0 + if verbose + fprintf(['\t...Unvalid stopping rule for INSTA.',... + 'Using default relative tolerance = 1e-4\n']); + end + Niter = -1; + relthre=1e-4; + end + + % start ista + t0=1;t1=1; + w2=rls_primal_driver(XtX, Xty, n, 0); + w2(abs(w2)relthre*max([mean(abs(w1)),mean(abs(w2))]) + yk=w2+(t0-1)*(w2-w1)/t1; + w1=w2; + w2=(1-lambda*gamma*(1-inst_alpha))*yk+2*gamma*(Xty-XtX*yk)/n; + w2(abs(w2)relthre*max([mean(abs(w1)),mean(abs(w2))]) w1=w2; w2=(1-lambda*gamma*(1-inst_alpha))*w2+2*gamma*(Xty-XtX*w2)/n; @@ -54,5 +55,6 @@ end end + w=w2; end \ No newline at end of file From 5ca94284f7330388c1d0022d2c3e5962e5239223 Mon Sep 17 00:00:00 2001 From: curious21cn Date: Tue, 15 Dec 2015 23:54:23 -0500 Subject: [PATCH 6/6] Correct some typos for ISTA --- gurls/gurls_defopt.m | 10 +++++----- gurls/optimizers/rls_fista.m | 25 +++++++++++------------ gurls/optimizers/rls_ista.m | 22 ++++++++++----------- gurls/paramsel/paramsel_hoista.m | 34 ++++++++++++++++---------------- gurls/utils/rls_fista_driver.m | 6 +++--- gurls/utils/rls_ista_driver.m | 6 +++--- 6 files changed, 51 insertions(+), 52 deletions(-) diff --git a/gurls/gurls_defopt.m b/gurls/gurls_defopt.m index c614cde..c534693 100644 --- a/gurls/gurls_defopt.m +++ b/gurls/gurls_defopt.m @@ -77,9 +77,9 @@ opt.newprop( 'seq', {}); opt.newprop( 'process', {}); %% ISTA options - opt.newprop( 'INSTAalpha',1); - opt.newprop( 'INSTAniter',inf); - opt.newprop( 'INSTArelthre',1e-4); - opt.newprop( 'INSTASparsitylvl',inf); - opt.newprop( 'INSTASAccuReq',0); + opt.newprop( 'ISTAalpha',1); + opt.newprop( 'ISTAniter',inf); + opt.newprop( 'ISTArelthre',1e-4); + opt.newprop( 'ISTASparsitylvl',inf); + opt.newprop( 'ISTASAccuReq',0); end diff --git a/gurls/optimizers/rls_fista.m b/gurls/optimizers/rls_fista.m index 1d4a306..bc1cdb0 100644 --- a/gurls/optimizers/rls_fista.m +++ b/gurls/optimizers/rls_fista.m @@ -1,7 +1,6 @@ function [cfr] = rls_fista (X, y, opt) - -% rls_insta(X,y,opt) -% computes a classifier for elastic nerwork using ISTA. +% rls_fista(X,y,opt) +% computes a classifier for elastic nerwork using FISTA. % The regularization parameter is set to the one found in opt.paramsel. % In case of multiclass problems, the regularizers need to be combined with the opt.singlelambda function. % @@ -12,7 +11,7 @@ % fields with default values set through the defopt function: % - singlelambda % fields with default values set through the defopt function: -% - insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) +% - ISTAalpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) % - niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) % - relthre (relative convergence threshold for iteration to stop) % @@ -29,26 +28,26 @@ n = size(y,1); % load in parameters alpha -if isprop(opt,'INSTAalpha') - INSTAalpha=opt.INSTAalpha; - if INSTAalpha <= 0 || INSTAalpha > 1 +if isprop(opt,'ISTAalpha') + ISTAalpha=opt.ISTAalpha; + if ISTAalpha <= 0 || ISTAalpha > 1 error('Invalid alpha'); end else if opt.verbose fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); - INSTAalpha=1; + ISTAalpha=1; end end % load in number of iterations or relative tolerance Niter=inf; relthre=1e-4; -if isprop(opt, 'INSTAniter') - Niter=opt.INSTAniter; +if isprop(opt, 'ISTAniter') + Niter=opt.ISTAniter; end -if isprop(opt, 'INSTArelthre') - relthre=opt.INSTArelthre; +if isprop(opt, 'ISTArelthre') + relthre=opt.ISTArelthre; end % check if matrices XtX and Xty have been previously computed during @@ -68,7 +67,7 @@ %redo OLR based on non-sparsy components -w = rls_fista_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt.verbose); +w = rls_fista_driver( XtX, Xty, n, lambda,ISTAalpha,Niter,relthre,opt.verbose); cfr.IndexFlag = ~~(w); % Xs=X(:,~~w); diff --git a/gurls/optimizers/rls_ista.m b/gurls/optimizers/rls_ista.m index 51c9452..9a60244 100644 --- a/gurls/optimizers/rls_ista.m +++ b/gurls/optimizers/rls_ista.m @@ -1,6 +1,6 @@ function [cfr] = rls_ista (X, y, opt) -% rls_insta(X,y,opt) +% rls_ista(X,y,opt) % computes a classifier for elastic nerwork using ISTA. % The regularization parameter is set to the one found in opt.paramsel. % In case of multiclass problems, the regularizers need to be combined with the opt.singlelambda function. @@ -12,7 +12,7 @@ % fields with default values set through the defopt function: % - singlelambda % fields with default values set through the defopt function: -% - insta_alpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) +% - ISTAalpha (paramters for balancing l1-norm and l-2 norm. 1 for LASSO and 0 for ridge) % - niter (maximun number for iteration. Set to either negative number or inf for using threshold rule only) % - relthre (relative convergence threshold for iteration to stop) % @@ -29,26 +29,26 @@ n = size(y,1); % load in parameters alpha -if isprop(opt,'INSTAalpha') - INSTAalpha=opt.INSTAalpha; - if INSTAalpha <= 0 || INSTAalpha > 1 +if isprop(opt,'ISTAalpha') + ISTAalpha=opt.ISTAalpha; + if ISTAalpha <= 0 || ISTAalpha > 1 error('Invalid alpha'); end else if opt.verbose fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); - INSTAalpha=1; + ISTAalpha=1; end end % load in number of iterations or relative tolerance Niter=inf; relthre=1e-4; -if isprop(opt, 'INSTAniter') - Niter=opt.INSTAniter; +if isprop(opt, 'ISTAniter') + Niter=opt.ISTAniter; end -if isprop(opt, 'INSTArelthre') - relthre=opt.INSTArelthre; +if isprop(opt, 'ISTArelthre') + relthre=opt.ISTArelthre; end % check if matrices XtX and Xty have been previously computed during @@ -66,7 +66,7 @@ %redo OLR based on non-sparsy components -w = rls_ista_driver( XtX, Xty, n, lambda,INSTAalpha,Niter,relthre,opt.verbose); +w = rls_ista_driver( XtX, Xty, n, lambda,ISTAalpha,Niter,relthre,opt.verbose); cfr.IndexFlag = ~~(w); % Xs=X(:,~~w); diff --git a/gurls/paramsel/paramsel_hoista.m b/gurls/paramsel/paramsel_hoista.m index ae1a412..2fca682 100644 --- a/gurls/paramsel/paramsel_hoista.m +++ b/gurls/paramsel/paramsel_hoista.m @@ -1,6 +1,6 @@ function vout = paramsel_hoista(X, y, opt) -% paramsel_hoinsta(X,Y,OPT) -% Performs parameter selection INSTA for elastic net is used. +% paramsel_hoista(X,Y,OPT) +% Performs parameter selection ISTA for elastic net is used. % The hold-out approach is used. % The performance measure specified by opt.hoperf is maximized. % @@ -43,22 +43,22 @@ % load in number of iterations or relative tolerance Niter=inf; relthre=1e-4; -if isprop(opt, 'INSTAniter') - Niter=opt.INSTAniter; +if isprop(opt, 'ISTAniter') + Niter=opt.ISTAniter; end -if isprop(opt, 'INSTArelthre') - relthre=opt.INSTArelthre; +if isprop(opt, 'ISTArelthre') + relthre=opt.ISTArelthre; end % load in alpha for elastic net -if isprop(opt,'INSTAalpha') - INSTAalpha=opt.INSTAalpha; - if INSTAalpha <= 0 || INSTAalpha > 1 +if isprop(opt,'ISTAalpha') + ISTAalpha=opt.ISTAalpha; + if ISTAalpha <= 0 || ISTAalpha > 1 error('Invalid alpha'); end else if opt.verbose fprintf('\t...alpha not defined. Use default value alpha=1 for LASSO\n'); - INSTAalpha=1; + ISTAalpha=1; end end @@ -69,8 +69,8 @@ T = size(y,2); % Verify the parameter selection constraint is defined proper -opt.INSTASparsitylvl=ceil(opt.INSTASparsitylvl); -if (opt.INSTASAccuReq>0&&opt.INSTASparsitylvl0&&opt.ISTASparsitylvlopt.INSTASparsitylvl),:)=-1; + if opt.ISTASparsitylvlopt.ISTASparsitylvl),:)=-1; [~, idx] = max(apt,[],1); else - Sparsity(min(ap,[],2)