done?

README.txt

@@ -0,0 +1,19 @@
+
+/data - pulled images and spreadsheet of data
+/descriptor - functions for extracting descriptors
+/distance - functions fo measuring distance between descriptors
+/util - util functions such as toGreyscale and EVD
+
+There are two types of script, ones that run a category response once (cvpr_visualsearch_*) and ones that iteratively 
+generate new descriptors to run queries on (parameter_*)
+
+_query_set operates using either L1 or L2 norm on the query set
+_pca generates an eigenmodel from the descriptors and computes mahalanobis distance
+_rand_image picks a random query image from each category to iterate over, no results from this script are in the paper
+
+
+The cvpr_visualsearch_* scripts load descriptors from folders and perform a category response test on them.
+
+The parameter_* scripts were used to generate iterative parameter results for descriptors.
+Effectively the query code from the cvpr_visualsearch_* files have been prefaced with descriptor generators that as a whole
+iterate over parameters instead of loading them from files.

cvpr_computedescriptors.m

@@ -23,10 +23,10 @@ OUT_FOLDER = 'descriptors';
 %% the idea is all your descriptors are in individual folders - within
 %% the folder specified as 'OUT_FOLDER'.
 % OUT_SUBFOLDER='avgRGB';
-% OUT_SUBFOLDER='globalRGBhisto';
+OUT_SUBFOLDER='globalRGBhisto';
 % OUT_SUBFOLDER='spatialColour';
 % OUT_SUBFOLDER='spatialTexture';
-OUT_SUBFOLDER='spatialColourTexture';
+% OUT_SUBFOLDER='spatialColourTexture';
 
 allfiles=dir (fullfile([DATASET_FOLDER,'/Images/*.bmp']));
 for filenum=1:length(allfiles)
@@ -39,10 +39,10 @@ for filenum=1:length(allfiles)
     
     %% EXTRACT FUNCTION
 %     F=extractAvgRGB(img);
-%     F=extractGlobalColHist(img, 2, 1, 8, 0.05);
+    F=extractGlobalColHist(img, 5);
-%     F=extractSpatialColour(img, 14, 4);
+%     F=extractSpatialColour(img, 4, 4);
 %     F=extractSpatialTexture(img, 4, 4, 7, 0.09);
-    F=extractSpatialColourTexture(img, 4, 4, 7, 0.09);
+%     F=extractSpatialColourTexture(img, 4, 4, 7, 0.09);
     save(fout,'F');
     toc
 end

cvpr_visualsearch_pca.m

@@ -0,0 +1,263 @@
+%% EEE3032 - Computer Vision and Pattern Recognition (ee3.cvpr)
+%%
+%% cvpr_visualsearch.m
+%% Skeleton code provided as part of the coursework assessment
+%%
+%% This code will load in all descriptors pre-computed (by the
+%% function cvpr_computedescriptors) from the images in the MSRCv2 dataset.
+%%
+%% It will pick a descriptor at random and compare all other descriptors to
+%% it - by calling cvpr_compare.  In doing so it will rank the images by
+%% similarity to the randomly picked descriptor.  Note that initially the
+%% function cvpr_compare returns a random number - you need to code it
+%% so that it returns the Euclidean distance or some other distance metric
+%% between the two descriptors it is passed.
+%%
+%% (c) John Collomosse 2010  (J.Collomosse@surrey.ac.uk)
+%% Centre for Vision Speech and Signal Processing (CVSSP)
+%% University of Surrey, United Kingdom
+
+close all;
+clear all;
+
+%% Edit the following line to the folder you unzipped the MSRCv2 dataset to
+DATASET_FOLDER = 'dataset';
+
+%% Folder that holds the results...
+DESCRIPTOR_FOLDER = 'descriptors';
+%% and within that folder, another folder to hold the descriptors
+%% we are interested in working with
+% DESCRIPTOR_SUBFOLDER='avgRGB';
+DESCRIPTOR_SUBFOLDER='globalRGBhisto';
+% DESCRIPTOR_SUBFOLDER='spatialColour';
+% DESCRIPTOR_SUBFOLDER='spatialTexture';
+% DESCRIPTOR_SUBFOLDER='spatialColourTexture';
+
+CATEGORIES = ["Farm Animal" 
+    "Tree"
+    "Building"
+    "Plane"
+    "Cow"
+    "Face"
+    "Car"
+    "Bike"
+    "Sheep"
+    "Flower"
+    "Sign"
+    "Bird"
+    "Book Shelf"
+    "Bench"
+    "Cat"
+    "Dog"
+    "Road"
+    "Water Features"
+    "Human Figures"
+    "Coast"
+    ];
+
+QUERY_INDEXES=[301 358 384 436 447 476 509 537 572 5 61 80 97 127 179 181 217 266 276 333];
+
+% 1_10 2_16 3_12 4_4 5_15 6_14 7_17 8_15 9_1 10_14 11_8 12_26 13_10 14_10
+% 15_8 16_10 17_16 18_5 19_15 20_12
+
+
+%% 1) Load all the descriptors into "ALLFEAT"
+%% each row of ALLFEAT is a descriptor (is an image)
+
+ALLFEAT=[];
+ALLFILES=cell(1,0);
+ALLCATs=[];
+ctr=1;
+allfiles=dir (fullfile([DATASET_FOLDER,'/Images/*.bmp']));
+for filenum=1:length(allfiles)
+    fname=allfiles(filenum).name;
+    
+    %identify photo category for PR calculation
+    split_string = split(fname, '_');
+    ALLCATs(filenum) = str2double(split_string(1));
+    
+    imgfname_full=([DATASET_FOLDER,'/Images/',fname]);
+    img=double(imread(imgfname_full))./255;
+    thesefeat=[];
+    featfile=[DESCRIPTOR_FOLDER,'/',DESCRIPTOR_SUBFOLDER,'/',fname(1:end-4),'.mat'];%replace .bmp with .mat
+    load(featfile,'F');
+    ALLFILES{ctr}=imgfname_full;
+    ALLFEAT=[ALLFEAT ; F];
+    ctr=ctr+1;
+end
+
+% get counts for each category for PR calculation
+CAT_HIST = histogram(ALLCATs).Values;
+CAT_TOTAL = length(CAT_HIST);
+
+NIMG=size(ALLFEAT,1);           % number of images in collection
+    
+descriptor_list = ALLFEAT;
+
+confusion_matrix = zeros(CAT_TOTAL);
+
+all_precision = [];
+all_recall = [];
+
+AP_values = zeros([1, CAT_TOTAL]);
+for iteration=1:CAT_TOTAL
+    
+    %% 2) Pick an image at random to be the query
+    queryimg=QUERY_INDEXES(iteration);    % index of a random image
+    
+    %% 3) Compute EigenModel
+    E = getEigenModel(descriptor_list);
+    E = deflateEigen(E, 0.986);
+    
+    %% 4) Project data to lower dimensionality
+    descriptor_list=descriptor_list-repmat(E.org,size(descriptor_list,1),1);
+    descriptor_list=((E.vct')*(descriptor_list'))';
+
+    %% 5) Compute the distance of image to the query
+    dst=[];
+    for i=1:NIMG
+        candidate=descriptor_list(i,:);
+        query=descriptor_list(queryimg,:);
+
+        category=ALLCATs(i);
+
+        %% COMPARE FUNCTION
+        thedst=compareMahalanobis(E, query, candidate);
+        dst=[dst ; [thedst i category]];
+    end
+    dst=sortrows(dst,1);  % sort the results
+
+    %% 6) Calculate PR
+    precision_values=zeros([1, NIMG-1]);
+    recall_values=zeros([1, NIMG-1]);
+
+    correct_at_n=zeros([1, NIMG-1]);
+
+    query_row = dst(1,:);
+    query_category = query_row(1,3);
+%     if query_category ~= iteration
+%         dst
+%     end
+    fprintf('category was %s\n', CATEGORIES(query_category))
+    
+    dst = dst(2:NIMG, :);
+    
+    %calculate PR for each n
+    for i=1:size(dst, 1)
+        % NIMG-1 and j iterator variable is in order to skip calculating for query image
+        
+        rows = dst(1:i, :);
+
+        correct_results = 0;
+        incorrect_results = 0;
+
+        if i > 1    
+            for n=1:i - 1
+                row = rows(n, :);
+                category = row(3);
+
+                if category == iteration
+                    correct_results = correct_results + 1;
+                else
+                    incorrect_results = incorrect_results + 1;
+                end
+
+            end
+        end
+
+        % LAST ROW
+        row = rows(i, :);
+        category = row(3);
+
+        if category == iteration
+            correct_results = correct_results + 1;
+            correct_at_n(i) = 1;
+        else
+            incorrect_results = incorrect_results + 1;
+        end
+
+        precision = correct_results / i;
+        recall = correct_results / (CAT_HIST(1,iteration) - 1);
+
+        precision_values(i) = precision;
+        recall_values(i) = recall;
+    end
+
+
+    %% 7) calculate AP
+    average_precision = sum(precision_values .* correct_at_n) / CAT_HIST(1,iteration);
+    AP_values(iteration) = average_precision;
+    
+ 
+    all_precision = [all_precision ; precision_values];
+    all_recall = [all_recall ; recall_values];
+    
+
+    %% 6) plot cumulative PR curve
+%     figure(1)
+%     plot(recall_values, precision_values,'LineWidth',1.5);
+%     hold on;
+%     title('PR Curve');
+%     xlabel('Recall');
+%     ylabel('Precision');
+%     xlim([0 1]);
+%     ylim([0 1]);
+    
+    
+    %% 8) Visualise the results and Populate confusion matrix
+    %% These may be a little hard to see using imgshow
+    %% If you have access, try using imshow(outdisplay) or imagesc(outdisplay)
+    
+    SHOW=25; % Show top 25 results
+    dst=dst(1:SHOW,:);
+    outdisplay=[];
+    for i=1:size(dst,1)
+       img=imread(ALLFILES{dst(i,2)});
+       img=img(1:2:end,1:2:end,:); % make image a quarter size
+       img=img(1:81,:,:); % crop image to uniform size vertically (some MSVC images are different heights)
+       outdisplay=[outdisplay img];
+       
+       %populate confusion matrix
+       confusion_matrix(dst(i,3), iteration) = confusion_matrix(dst(i,3), iteration) + 1;
+    end
+%     figure(3)
+%     imgshow(outdisplay);
+%     axis off;
+
+end
+
+%% 9) Plot average PR curve
+figure(4)
+mean_precision = mean(all_precision);
+mean_recall = mean(all_recall);
+plot(mean_recall, mean_precision,'LineWidth',5);
+title('Spatial Colour and Texture Average PR with PCA (4x3, 7 bins, thresh. 0.09)');
+xlabel('Average Recall');
+ylabel('Average Precision');
+xlim([0 1]);
+ylim([0 1]);
+
+%% 11) normalise confusion matrix
+figure(5)
+norm_confusion_matrix = confusion_matrix ./ sum(confusion_matrix, 'all');
+cm = confusionchart(confusion_matrix, CATEGORIES, 'Normalization', 'column-normalized');
+cm.Title = 'Spatial Colour and Texture Confusion Matrix with PCA (4x3, 7 bins, thresh. 0.09)';
+xlabel('Query Classification');
+ylabel('Ground Truth');
+
+%% 12) Calculate MAP
+% figure(4)
+% histogram(AP_values);
+% title('Average Precision Distribution');
+% ylabel('Count');
+% xlabel('Average Precision');
+% xlim([0, 1]);
+
+MAP = mean(AP_values)
+AP_sd = std(AP_values);
+
+% figure(2)
+% plot(1:CAT_TOTAL, AP_values);
+% title('Average Precision Per Run');
+% xlabel('Run');
+% ylabel('Average Precision');

cvpr_visualsearch_query_set.m

@@ -112,7 +112,7 @@ for iteration=1:CAT_TOTAL
         category=ALLCATs(i);
 
         %% COMPARE FUNCTION
-        thedst=compareEuclidean(query, candidate);
+        thedst=compareL1(query, candidate);
         dst=[dst ; [thedst i category]];
     end
     dst=sortrows(dst,1);  % sort the results
@@ -208,7 +208,7 @@ for iteration=1:CAT_TOTAL
        outdisplay=[outdisplay img];
        
        %populate confusion matrix
-       confusion_matrix(query_category, dst(i,3)) = confusion_matrix(query_category, dst(i,3)) + 1;
+       confusion_matrix(dst(i,3), iteration) = confusion_matrix(dst(i,3), iteration) + 1;
     end
 %     figure(3)
 %     imgshow(outdisplay);
@@ -221,14 +221,19 @@ figure(4)
 mean_precision = mean(all_precision);
 mean_recall = mean(all_recall);
 plot(mean_recall, mean_precision,'LineWidth',5);
-title('Spatial Texture Average PR (4x4, bin=7, thresh.=0.09)');
+title('Spatial Colour and Texture Average PR (4x3, 7 bins, thresh. 0.09)');
 xlabel('Average Recall');
 ylabel('Average Precision');
 xlim([0 1]);
 ylim([0 1]);
 
 % normalise confusion matrix
+figure(5)
 norm_confusion_matrix = confusion_matrix ./ sum(confusion_matrix, 'all');
+cm = confusionchart(confusion_matrix, CATEGORIES, 'Normalization', 'column-normalized');
+cm.Title = 'Global Colour Histogram Confusion Matrix, 5 bins';
+xlabel('Query Classification');
+ylabel('Ground Truth');
 
 %% 8 Calculate MAP
 % figure(4)

descriptor/extractGlobalColHist.m

@@ -1,6 +1,4 @@
-function F=extractGlobalColHist(img)
+function F=extractGlobalColHist(img, divs)
-
-divs = 5;
 
 qimg = floor(img .* divs);
 bin = qimg(:,:,1) * divs^2 + qimg(:,:,2) * divs^1 + qimg(:,:,3);

parameter_iter_pca.m

@@ -25,13 +25,6 @@ DATASET_FOLDER = 'dataset';
 
 %% Folder that holds the results...
 DESCRIPTOR_FOLDER = 'descriptors';
-%% and within that folder, another folder to hold the descriptors
-%% we are interested in working with
-% DESCRIPTOR_SUBFOLDER='avgRGB';
-% DESCRIPTOR_SUBFOLDER='globalRGBhisto';
-% DESCRIPTOR_SUBFOLDER='spatialColour';
-% DESCRIPTOR_SUBFOLDER='spatialTexture';
-DESCRIPTOR_SUBFOLDER='spatialColourTexture';
 
 CATEGORIES = ["Farm Animal" 
     "Tree"
@@ -60,6 +53,9 @@ QUERY_INDEXES=[301 358 384 436 447 476 509 537 572 5 61 80 97 127 179 181 217 26
 % 1_10 2_16 3_12 4_4 5_15 6_14 7_17 8_15 9_1 10_14 11_8 12_26 13_10 14_10
 % 15_8 16_10 17_16 18_5 19_15 20_12
 
+map=[];
+deflate_energy = [0:0.001:0.2];
+for var_iter=28:28%size(deflate_energy, 2)
     
 %% 1) Load all the descriptors into "ALLFEAT"
 %% each row of ALLFEAT is a descriptor (is an image)
@@ -68,22 +64,45 @@ ALLFEAT=[];
 ALLFILES=cell(1,0);
 ALLCATs=[];
 ctr=1;
+% allfiles=dir (fullfile([DATASET_FOLDER,'/Images/*.bmp']));
+% for filenum=1:length(allfiles)
+%     fname=allfiles(filenum).name;
+%     
+%     %identify photo category for PR calculation
+%     split_string = split(fname, '_');
+%     ALLCATs(filenum) = str2double(split_string(1));
+%     
+%     imgfname_full=([DATASET_FOLDER,'/Images/',fname]);
+%     img=double(imread(imgfname_full))./255;
+%     thesefeat=[];
+%     featfile=[DESCRIPTOR_FOLDER,'/',DESCRIPTOR_SUBFOLDER,'/',fname(1:end-4),'.mat'];%replace .bmp with .mat
+%     load(featfile,'F');
+%     ALLFILES{ctr}=imgfname_full;
+%     ALLFEAT=[ALLFEAT ; F];
+%     ctr=ctr+1;
+% end
+
 allfiles=dir (fullfile([DATASET_FOLDER,'/Images/*.bmp']));
 for filenum=1:length(allfiles)
     fname=allfiles(filenum).name;
+%     fprintf('Processing file %d/%d - %s\n',filenum,length(allfiles),fname);
+%     tic;
+    imgfname_full=([DATASET_FOLDER,'/Images/',fname]);
+    img=double(imread(imgfname_full))./255;
     
     %identify photo category for PR calculation
     split_string = split(fname, '_');
     ALLCATs(filenum) = str2double(split_string(1));
     
-    imgfname_full=([DATASET_FOLDER,'/Images/',fname]);
+    %% EXTRACT FUNCTION
-    img=double(imread(imgfname_full))./255;
+%     F=extractAvgRGB(img);
-    thesefeat=[];
+    F=extractGlobalColHist(img, var_iter);
-    featfile=[DESCRIPTOR_FOLDER,'/',DESCRIPTOR_SUBFOLDER,'/',fname(1:end-4),'.mat'];%replace .bmp with .mat
+%     F=extractSpatialColour(img, r, c);
-    load(featfile,'F');
+%     F=extractSpatialTexture(img, r, c, 7, 0.09);
-    ALLFILES{ctr}=imgfname_full;
+%     F=extractSpatialColourTexture(img, r, c, 7, 0.09);
+%     toc
+    
     ALLFEAT=[ALLFEAT ; F];
-    ctr=ctr+1;
 end
 
 % get counts for each category for PR calculation
@@ -92,12 +111,6 @@ CAT_TOTAL = length(CAT_HIST);
 
 NIMG=size(ALLFEAT,1);           % number of images in collection
     
-
-
-map=[];
-deflate_energy = [0:0.001:0.3];
-for var_iter=1:size(deflate_energy, 2)
-    
 descriptor_list = ALLFEAT;
 
 confusion_matrix = zeros(CAT_TOTAL);
@@ -113,7 +126,7 @@ for iteration=1:CAT_TOTAL
     
     %% 3) Compute EigenModel
     E = getEigenModel(descriptor_list);
-    E = deflateEigen(E, 1-deflate_energy(var_iter));
+    E = deflateEigen(E, 0.986);
     
     %% 4) Project data to lower dimensionality
     descriptor_list=descriptor_list-repmat(E.org,size(descriptor_list,1),1);
@@ -218,13 +231,13 @@ for iteration=1:CAT_TOTAL
 %     dst=dst(1:SHOW,:);
 %     outdisplay=[];
 %     for i=1:size(dst,1)
-%        img=imread(ALLFILES{dst(i,2)});
+% %        img=imread(ALLFILES{dst(i,2)});
-%        img=img(1:2:end,1:2:end,:); % make image a quarter size
+% %        img=img(1:2:end,1:2:end,:); % make image a quarter size
-%        img=img(1:81,:,:); % crop image to uniform size vertically (some MSVC images are different heights)
+% %        img=img(1:81,:,:); % crop image to uniform size vertically (some MSVC images are different heights)
-%        outdisplay=[outdisplay img];
+% %        outdisplay=[outdisplay img];
 %        
 %        %populate confusion matrix
-%        confusion_matrix(query_category, dst(i,3)) = confusion_matrix(query_category, dst(i,3)) + 1;
+%        confusion_matrix(dst(i,3), iteration) = confusion_matrix(dst(i,3), iteration) + 1;
 %     end
 %     figure(3)
 %     imgshow(outdisplay);

parameter_iter_query_set.m

@@ -55,8 +55,8 @@ QUERY_INDEXES=[301 358 384 436 447 476 509 537 572 5 61 80 97 127 179 181 217 26
 
 map = [];
 
-for r=9:9
+for r=39:45
-    for c=1:8
+%     for c=1:8
 
 %% 1) Load all the descriptors into "ALLFEAT"
 %% each row of ALLFEAT is a descriptor (is an image)
@@ -79,9 +79,9 @@ for filenum=1:length(allfiles)
     
     %% EXTRACT FUNCTION
 %     F=extractAvgRGB(img);
-%     F=extractGlobalColHist(img);
+    F=extractGlobalColHist(img, r);
 %     F=extractSpatialColour(img, r, c);
-    F=extractSpatialTexture(img, r, c, 7, 0.09);
+%     F=extractSpatialTexture(img, r, c, 7, 0.09);
 %     F=extractSpatialColourTexture(img, r, c, 7, 0.09);
 %     toc
     
@@ -114,7 +114,7 @@ for iteration=1:CAT_TOTAL
         category=ALLCATs(i);
 
         %% COMPARE FUNCTION
-        thedst=compareEuclidean(query, candidate);
+        thedst=compareL1(query, candidate);
         dst=[dst ; [thedst i category]];
     end
     dst=sortrows(dst,1);  % sort the results
@@ -249,11 +249,11 @@ MAP = mean(AP_values);
 % xlabel('Run');
 % ylabel('Average Precision');
 
-fprintf('%i,%i %i\n', r, c, MAP);
+% fprintf('%i,%i %i\n', r, c, MAP);
-% fprintf('%i,%i\n', r, MAP);
+fprintf('%i,%i\n', r, MAP);
 
-map(r, c) = MAP;
+% map(r, c) = MAP;
-% map(b) = MAP;
+map(r) = MAP;
 
-    end
+%     end
 end