LHS=cos^2(pi/7)+cos^2((2pi)/7)+cos^2((3pi)/7)
=cos^2(pi/7)+cos^2((2pi)/7)+cos^2(pi-(4pi)/7)
=cos^2(pi/7)+cos^2((2pi)/7)+cos^2((4pi)/7)
using formula cos^2theta=1/2(1+cos2theta)
=1/2(1+cos((2pi)/7))+1/2(1+cos((4pi)/7))+ 1/2(1+cos((8pi)/7))
=3/2+1/2(cos(2pi/7)+cos(4pi/7)+cos((8pi)/7))
=3/2+1/(4sin(pi/7)) (2sin(pi/7)cos((2pi)/7)+2sin(pi/7)cos((4pi)/7)+2sin(pi/7)cos((8pi)/7))
=3/2+1/(4sin(pi/7)) (cancel(sin((3pi)/7))-sin(pi/7)+sin((5pi)/7)-cancel(sin((3pi)/7))+sin((9pi)/7)-sin((7pi)/7))
=3/2+1/(4sin(pi/7)) (-sin(pi/7)+sin(pi-(2pi)/7)+sin(pi+(2pi)/7)-0)
=3/2+1/(4sin(pi/7)) (-sin(pi/7)+cancel(sin((2pi)/7))-cancel(sin(2pi/7)))
=3/2+1/(4cancel(sin(pi/7))) (-cancel(sin(pi/7)))
=3/2-1/4=(6-1)/4=5/4
