This is largely assembled from existing examples. I'm sure we can find a way to break it, but this image is pretty forgiving. If the line detection is poor, or if there are multiple implied polygons, expect problems. I'm not sure how the reordering will behave with grossly nonconvex polygons.
% the image
inpict = imread('https://www.mathworks.com/matlabcentral/answers/uploaded_files/171466/image.jpeg');
inpict = im2gray(inpict);
BW = inpict<128; % reduce to a logical mask
%Find peaks in the Hough transform of the image.
[H,T,R] = hough(BW);
P = houghpeaks(H,5,'threshold',ceil(0.3*max(H(:))));
lines = houghlines(BW,T,R,P,'FillGap',5,'MinLength',7);
% plot the detected segments
imshow(inpict), hold on
for k = 1:numel(lines)
xy = [lines(k).point1; lines(k).point2];
plot(xy(:,1),xy(:,2),'LineWidth',2,'Color','green');
% Plot beginnings and ends of lines
plot(xy(1,1),xy(1,2),'x','LineWidth',2,'Color','yellow');
plot(xy(2,1),xy(2,2),'x','LineWidth',2,'Color','red');
end
% find intersections between segments
% see FEX #11837 (attached)
intlist = [];
for k1 = 1:numel(lines)-1
for k2 = k1 + 1:numel(lines)
xy1 = [lines(k1).point1; lines(k1).point2];
xy2 = [lines(k2).point1; lines(k2).point2];
[x y] = intersections(xy1(:,1), xy1(:,2), xy2(:,1), xy2(:,2));
xy = [x y];
if ~isempty(xy)
intlist = [intlist; xy]; %#ok<AGROW>
end
end
end
% reorder the vertices
% see Bruno's answer to question #429265
intlistc = mean(intlist,1);
P = intlist - intlistc;
[~,~,V] = svd(P,0);
[~,is] = sort(atan2(P*V(:,1),P*V(:,2)));
intlist = intlist(is([1:end 1]),:); % close the pgon
% plot the calculated polygon
figure
imshow(inpict); hold on
plot(intlist(:,1),intlist(:,2),'LineWidth',2,'Color','green')
See:
