MGM.10 The Spectral Theorem for Symmetric Matrices and the Holy Grail of Matrix Theory
 BASICS

Needs["LinearAlgebra`MatrixManipulation`"];
Needs["LinearAlgebra`Orthogonalization`"];
Needs["Graphics`Colors`"];
Off[General::"spell"];
Off[General::"spell1"];
Off[ParametricPlot3D::"ppcom"];
Off[ParametricPlot::"ppcom"];
Off[Plot::"plnr"];
SetOptions[Limit, Analytic -> True];
$Pre = Chop;

CMView = {2.7, 1.6, 1.2};

Vector Drawer

\(\(\( (*\ \(\(:\)\(Date : \ \ Copyright\ 2001\)\), \ 
      Math\ Everywhere, \ \(\(Inc\)\(.\)\)\ 
*) \)\(
\)\( (*
Original\ Conception\ to\ create\ a\ vector\ drawer\ is\ due\ to\ Bill\ Davis . 
Current\ code\ is\ by\ Bruce\ \(\(Carpenter\)\(.\)\)
*) \)\(
\)\( (*\ \ \(\(\(:\)\(Drawbacks : \ \ Since\ the\ vectors\ are\ drawn\ out\ of\)\)\ 

      \ \ \ \ \ context\), \ 
      the\ shapes\ of\ the\ arrowheads\ are\ only\ 
\ \ \ \ \ correct\ with\ AspectRatio\ set\ to\ \(\(Automatic\)\(.\)\)
*) \)\)\)

BeginPackage["Vector3D`","Graphics`Colors`"];

Axes3D::usage = "Axes3D[a, b] creates a Graphics3D object of cartesian axes with x, y, and z running from -a/3 to a, and with axes labels b units beyond the tips of the axes.  Axes3D[a] is Axes3D[a, a/8].";

Perpend::usage = "Perpend[a] returns a unit vector which is perpendicular to the vector a.";

Arrow::usage = "Arrow[a] produces a 2 or 3 dimensional vector from the origin to a. Arrow[a, Tail -> tail] gives a vector from tail to a.";
	
Vector::usage = "Vector[a, b] gives a 2D or 3D vector from a to b.";

ArrowHead::usage = "ArrowHead[a, vec] produces an arrowhead with its tip placed at the point a, pointing in the direction of the vector vec.";

Tail::usage = "Tail -> point puts the tail of the vector at point.";

Norm::usage = "Norm[vec] returns the Euclidean length of vec.";

Normalize::usage = "Normalize[vec] returns a unit vector in the direction of vec, provided vec is nonzero.";

Aperture::usage = "Aperture is the ratio of the radius of the base to the length of the head of a vector.";

TipSize::usage = "TipSize specifies an absolute size for the inner tip length of the head of a vector.";

TipRatio::usage = "TipRatio is the ratio of the inner length of the arrowhead to the length of the shaft of a vector.";

HeadRatio::usage = "HeadRatio is the ratio of the outer length of the arrowhead to the length of the shaft of a vector.";

HeadSize::usage = "HeadSize specifies an absolute size for the head of a vector.";

ScaleFactor::usage = "ScaleFactor specifies the amount to scale a vector in length and may be either a number or a pure function.";

ZeroVectorPointSize::usage = "ZeroVectorPointSize is the size of the point used to represent a zero vector.";

Options[Vector] = Options[Arrow] = Options[ArrowHead] =
{HeadRatio -> 0.18, TipRatio -> 0.14, Aperture -> 0.3, ScaleFactor -> 1, HeadType -> Polygon, ShaftQ -> True, EdgesQ -> True, VectorColor -> RGBColor[0,0,1], ShaftWidth -> 0.005, ZeroVectorPointSize -> 0.01};
		
Begin["`Private`"];

SetOptions[ParametricPlot,AspectRatio -> Automatic];
SetOptions[Plot,AspectRatio -> Automatic];
SetOptions[Graphics,AspectRatio -> Automatic];

Axes3D[u_,v_] := 
      Graphics3D[{
            {Blue,Line[{{-u/3,0,0},{u,0,0}}]},
                    Text["x",{u + v,0,0}],
            {Blue,Line[{{0,-u/3,0},{0,u,0}}]},
                    Text["y",{0, u + v,0}],
            {Blue,Line[{{0,0,-u/3},{0,0,u}}]},
                    Text["z",{0,0,u + v}]}]; 
Axes3D[u_] := Axes3D[u,u/8];
				
Norm[a_?VectorQ] := N[Sqrt[a.a]];

Normalize[a_?VectorQ] := If[a.a == 0, a, a/Norm[a]];

Perpend[a_?VectorQ] := Normalize[{a[[2]],-a[[1]]}]/;Length[a]==2;

Perpend[a_?VectorQ]:=
    Normalize[If[a[[1]]==0,{1,0,0},{a[[2]],-a[[1]],0}]]/;
      Length[a]==3;
      
Base3D = Table[{Cos[2 k Pi/8.], Sin[2 k Pi/8.]},{k,0,8}]; 

Vector[from_?VectorQ, to_?VectorQ, opts___] := Module[
   {sq, sw, ht, hs, hr, tr, ap, sf, vc, zvps, diff = to - from, tip},
   {sq, sw, ht, hs, hr, ts, tr, ap, sf, vc, zvps} = {ShaftQ, ShaftWidth, HeadType, HeadSize, HeadRatio, TipSize, TipRatio, Aperture, ScaleFactor, VectorColor, ZeroVectorPointSize} /. {opts} /. Options[Vector];
   If[from==to, Return[Graphics[{PointSize[zvps],vc,Point[from]}]]];
   diff = If[NumberQ[N[sf]], sf diff, sf[diff]];
   tip = from + diff;
   If[NumberQ[hs], hr = hs/Norm[diff]; tr = 0.75 hr];
   If[NumberQ[ts], tr = ts/Norm[diff]];
   Graphics[{If[sq,{vc,Thickness[sw],Line[{from,tip - tr diff}]},{}],
      With[{trans = ap hr Norm[diff] Perpend[diff]},
      {vc, Thickness[sw], ht[{tip,
                   (tip - hr diff + trans),
                   (tip - tr diff),
                   (tip - hr diff - trans),tip}]}]}]
] /; Length[from]==Length[to]==2

Vector[from_?VectorQ, to_?VectorQ, opts___] := Module[
   {sq, eq, sw, hs, hr, tr,ap, sf, vc, zvps, diff = to - from, tip},
   {sq, eq, sw, hs, hr, ap, sf, vc, zvps} = {ShaftQ, EdgesQ, ShaftWidth, HeadSize, HeadRatio, Aperture, ScaleFactor, VectorColor, ZeroVectorPointSize} /. {opts} /. Options[Vector];
   If[from==to, Return[Graphics3D[{PointSize[zvps],vc,Point[from]}]]];
   diff = If[NumberQ[N[sf]], sf diff, sf[diff]];
   tip = from + diff;
   If[NumberQ[hs], hr = hs/Norm[diff]];
   tr = hr/2;
   Graphics3D[{If[sq,{vc,Thickness[sw],Line[{from,tip-tr diff}]},{}], If[eq, {}, EdgeForm[]], {SurfaceColor[vc],
      With[{temp = Perpend[diff]},
      Polygon[Append[#,tip]]& /@ Partition[(tip - hr diff + #)&/@((ap hr Norm[diff] Base3D) . {temp,Normalize[Cross[temp,diff]]}),2,1]]}}]
] /; Length[from]==Length[to]==3


Arrow[a_, Tail->b_, opts___] := Vector[b,a+b,opts]

Arrow[a_, opts___] := Vector[Table[0,{Length[a]}],a,opts]

ArrowHead[a_, b_, opts___] := Vector[a-b,a,ShaftQ->False,opts];

End[];

EndPackage[];
Null

Graphics`Colors`GosiaGreen = 
        RGBColor[0,  0.392187,  0];

ThreeAxes[u,v]

ThreeAxes[u_, v_] := 
        Graphics3D[{{Blue, Line[{{-u, 0, 0}, {u, 0, 0}}]}, 
            Text["x", {u + v, 0, 0}], {Blue, 
              Line[{{0, -u, 0}, {0, u, 0}}]}, 
            Text["y", {0, u + v, 0}], {Blue, 
              Line[{{0, 0, 0}, {0, 0, u}}]}, 
            Text["z", {0, 0, u + v}]}];
ThreeAxes[u_] := ThreeAxes[u, (u/8)];
ThreeAxes::"usage" = "ThreeAxes[a,b] makes a standard cartesian \
axis graphics object with x, y, and z running from -a to a, and with axis \
labels b units beyond the tips of the axes.  ThreeAxes[a] is \
ThreeAxes[a,a/8].";

Needs["Statistics`ContinuousDistributions`"];

Clear[matrixplucker];
matrixplucker[expr_, 
        vars_] := Simplify[
          Outer[D, {expr}, 
            vars]][[1]]

B.1) The Spectral Theorem says that if A is a square symmetric matrix (A = (A^t)) hitting on hitdimD, then there is an orthonormal basis of hitdimD consisting of unit eigenvectors of A.

B.1.a.i) The Spectral Theorem says that if A is a square symmetric matrix (A = (A^t)) hitting on hitdimD, then there is an orthonormal basis of hitdimD consisting of unit eigenvectors of A.

Talk a little bit about the spectral theorem.

The spectral theorem says that if you have symmetric matrix A (this means A = (A^t)), then there is an orthonormal basis of hitdimD consisting of unit eigenvectors of A.
Try it out on a random symmetric matrix A:

hitdim = Random[Integer, {3, 5}];
Clear[b, i, j];
b[i_, j_] := Random[Real, {-4, 4}]
B = Table[b[i, j], {i, 1, hitdim}, {j, 1, hitdim}];
A  =  Transpose[B]  . B;
MatrixForm[A]

Let Mathematica calculate a linearly independent set of eigenvectors of A:

Clear[eigenvector, k];
eigenvector[
        k_]  := Eigenvectors[A][[k]]
Table[eigenvector[k], {k, 1, hitdim}]

This set of eigenvectors of A forms an orthonormal basis of hitdimD as the following calculation confirms:

Table[
      Table[eigenvector[j] . eigenvector[k], {k, 1, hitdim}], {j, 1, 
        hitdim}]

This output signals that if  k ≠ j, then eigenvector[k] is perpendicular to eigenvector[j] 
and
each of the calculated eigenvectors is a unit vector.
Try it for more random symmetric matrices:

hitdim = Random[Integer, {3, 10}];
Clear[b, i, j];
b[i_, j_] := Random[Real, {-4, 4}]
B = Table[b[i, j], {i, 1, hitdim}, {j, 1, hitdim}];
A  =  Transpose[B]  . B;
MatrixForm[A]
Clear[eigenvector, k];
eigenvector[k_]  := Eigenvectors[A][[k]]
Table[eigenvector[k], {k, 1, hitdim}];
Table[
      Table[eigenvector[j] . eigenvector[k], {k, 1, hitdim}], {j, 1, 
        hitdim}]

Rerun as much as you want to.

B.1.a.ii) Explanation of the Spectral Theorem

How about an explanation of where the Spectral Theorem comes from?

Hold your horses. 
You will get a full explanation later in this lesson.
Learning how to use the Spectral Theorem is more important than knowing its proof.

B.2) The Spectral Theorem as the holy grail of matrix theory:
If A is any matrix, then  (A^t) . A is symmetric.

For any matrix A, the Spectral theorem gives you an orthonormal basis 
           {(Subscript[X, 1]), (Subscript[X, 2]), (Subscript[X, 3]), (Subscript[X, 4]), ...., (Subscript[X, hitdim])} 
of hitdimD consisting of unit eigenvectors of (A^t) . A .
You can get the alignerframe, the stretch factors and the hangerframe for A directly from these vectors.

B.2.a.i) If A is any matrix, then (A^t) . A is symmetric

Explain this:
If A is any matrix, then (A^t) . A is symmetric.

Go with any matrix A and put B = (A^t) . A
Now note that:

(B^t) =  ((((A^t).A))^(t ))=  (A^t). ((((A^t).))^(t )) = (A^t). A = B.

B.2.a.ii) Using the Spectral Theorem to (A^t) . A to unlock the secrets of SVD analysis of A

For any matrix A, the Spectral theorem gives you an orthonormal basis 
           {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} 
of hitdimD consisting of unit eigenvectors of (A^t) . A.

Group and (if necessary relabel) these vectors into two groups:
        
Group I = {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} for which A.(Subscript[X, j]) ≠ {0,0,...0} for all j = 1,2,...,p.

Group 2 = {(Subscript[X, p]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} for which A.(Subscript[X, j]) = {0,0,...} 
      for all j = p+1,p+2, ...,hitdimD.
      
Facts of the matter:
▪ An SVD alignerframe for A,
          {alignerframe[1],alignerframe[2],.alignerframe[3], . .,alignerframe[p]}
 is 
            {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.

▪ The corresponding non-zero SVD stretch factors  for A,
         {stretch[1],stretch[2],stretch[3],....,stretch[p]}, 
are
         {||A . (Subscript[X, 1])||,||A . (Subscript[X, 2])||,||A . (Subscript[X, 3])||,....,||A . (Subscript[X, p])||}.

▪ The corresponding SVD hangerframe for A, 
       {hangerframe[1],hangerframe[2], . . .,hangerframe[p]}, 
is

{(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

Go with this matrix  A:

hitdim  =  8;
hangdim =  5;
A  =  
        Table[Random[Real, {-4, 4}], {i, 1, hangdim}, {j, 1, 
            hitdim}];
MatrixForm[A]

And use unit eigenvectors of (A^t) . A to generate the full SVD of A.

The Spectral theorem gives you an orthonormal basis 
           {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} 
of hitdimD consisting of unit eigenvectors of (A^t) . A. 
Here they are:

Clear[eigenvector, k];
eigenvector[
          k_]  :=  Eigenvectors[Transpose[A] . A][[k]];
ColumnForm[Table[eigenvector[k], {k, 1, hitdim}]]

Group and (if necessary relabel) these vectors into two groups:
        
Group I = {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} for which A.(Subscript[X, j]) ≠ {0,0,...0} for all j = 1,2,...,p.

Group 2 = {(Subscript[X, p]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} for which A.(Subscript[X, j]) = {0,0,...} 
      for all j = p+1,p+2, ...,hitdimD.
 Find out what p is:

ColumnForm[Table[A.eigenvector[k],{k, 1, hitdim}]]

So:

p  =  5

▪ An SVD alignerframe for A,
          {alignerframe[1],alignerframe[2],.alignerframe[3], . .,alignerframe[p]}
 is 
            {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}:

p  =  5;
alignerframe  =  Table[eigenvector[k], {k, 1, p}]

▪ The corresponding SVD stretch factors  for A,
         {stretch[1],stretch[2],stretch[3],....,stretch[p]}, 
are
         {||A . (Subscript[X, 1])||,||A . (Subscript[X, 2])||,||A . (Subscript[X, 3])||,....,||A . (Subscript[X, p])||}:

stretches = Table[Norm[A . eigenvector[k]],{k, 1, p}]

▪ The corresponding SVD hangerframe for A, 
       {hangerframe[1],hangerframe[2], . . .,hangerframe[p]}, 
is

{(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

hangerframe =Table[  (1/(Norm[A . eigenvector[k]]))A.eigenvector[k],{k, 1, p}]

Check it out:

aligner =  alignerframe;
MatrixForm[aligner]

stretcher  =  DiagonalMatrix[stretches];
MatrixForm[
      stretcher]

hanger  =  Transpose[hangerframe];
MatrixForm[hanger]

MatrixForm[hanger.stretcher.aligner]

MatrixForm[A]

On the money.
Big time math happens again.

B.2.a.iii) Explanations

For any matrix A, the Spectral theorem gives you an orthonormal basis 
           {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} 
of hitdimD consisting of unit eigenvectors of (A^t) . A.

Group and (if necessary relabel) these vectors into two groups:
        
Group I = {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} for which A.(Subscript[X, j]) ≠ {0,0,...0} for all j = 1,2,...,p.

Group 2 = {(Subscript[X, p]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} for which A.(Subscript[X, j]) = {0,0,...} 
      for all j = p+1,p+2, ...,hitdimD.
      
Facts of the matter:
▪ An SVD alignerframe for A,
          {alignerframe[1],alignerframe[2],.alignerframe[3], . .,alignerframe[p]}
 is 
            {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.

▪ The corresponding SVD stretch factors  for A,
         {stretch[1],stretch[2],stretch[3],....,stretch[p]}, 
are
         {||A . (Subscript[X, 1])||,||A . (Subscript[X, 2])||,||A . (Subscript[X, 3])||,....,||A . (Subscript[X, p])||}.

▪ The corresponding SVD hangerframe for A, 
       {hangerframe[1],hangerframe[2], . . .,hangerframe[p]}, 
is

{(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

▪ And when you go with these assignments, you are fully guaranteed that
    A.X =  (∑_(k = 1)^p)stretch[k] (X.alignerframe[k]) hangerframe[k] 
  for any X in hitdimD

Explain why all this works.

This is a little bit of mathematical bean-counting.

▪ Why the advertised aligner fame {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}  is a perpendicular frame.

Reason:The Spectral theorem gives you an orthonormal basis 
{(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} is an orthonormal basis (perpendicular frame) 
of hitdimD consisting of unit eigenvectors of (A^t) . A. 
So {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} is a perpendicular frame

▪ Why the advertised stretch factors
             stretch[k] = ||A . (Subscript[X, k])||  for k = 1,2,. . .,p 
 are all positive.

Reason:
      stretch[k] = ||A . (Subscript[X, k])|| for k = 1,2,. . .,p are all positive
 because
     A . (Subscript[X, k]) ≠ {0,0, . . . ,0} for k = 1,2,. . .,p

▪ Why the vectors in the advertised hanger frame

{(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

are all unit vectors.

Go with any k with k = 1,2,. . .,p, and look at

and look at

( (1/(||A . (Subscript[X, k])||))A.(Subscript[X, k])).  ( (1/(||A . (Subscript[X, k])||))A.(Subscript[X, k]))

=   \(1\/\(\(||\)\(A . X\_k\)\( || \^2\)\)\) (A. (Subscript[X, k ])). (A.(Subscript[X, k ]))

=     \(1\/\(\(|\)\(\(A . X\_k\)\( || \^2\)\)\)\) ||A.(Subscript[X, k ]) \( || \^2\)

= 1.

▪ Why the vectors in the advertised hanger frame

{(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

are orthonormal (perpendicular to each other).

Go with any k with k = 1,2,. . .,p, and any j with j = 1,2,. . .,p and j ≠ p

and look at

( (1/(||A . (Subscript[X, k])||))A.(Subscript[X, k])).  ( (1/(||A . (Subscript[X, j])||))A.(Subscript[X, j]))

=   \(1\/\(\(\(||\)\(A . X\_k\)\(||\)\(\ \ \)\(||\)\(A . X\_j\)\(||\)\)\(|\)\)\) (A. (Subscript[X, k ])). (A.(Subscript[X, j ]))

=   (1/(||A . (Subscript[X, k])||  ||A . 
                (Subscript[X, j])||| )) ( (Subscript[X, k ])). ((A^t)A.(Subscript[X, j ]))

=   (1/(||A . (Subscript[X, k])||  ||A . 
                (Subscript[X, j])|||)) ( (Subscript[X, k ])).(eigenvalue (Subscript[X, j ]))

=   (eigenvalue/(||A . (Subscript[X, k])||  ||A . (Subscript[X, j])||)) ( (Subscript[X, k ])).((Subscript[X, j ]))

=   (eigenvalue/(||A . (Subscript[X, k])||  ||A . (Subscript[X, j])||)) 0  = 0.

because (Subscript[X, j]) is perpendicular to (Subscript[X, k ])

▪ Why when you go with
 {alignerframe[1],alignerframe[2],.alignerframe[3], . .,alignerframe[p]}

=  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.

and
         {stretch[1],stretch[2],stretch[3],....,stretch[p]}

= {||A . (Subscript[X, 1])||,||A . (Subscript[X, 2])||,||A . (Subscript[X, 3])||,....,||A . (Subscript[X, p])||}.

and
       {hangerframe[1],hangerframe[2], . . .,hangerframe[p]},

=  {(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

you are guaranteed that
                A.alignerframe[k] = stretch[k] hangerframe[k] 
for all k = 1,2,3,. . .,p,

Because alignerframe[k] = (Subscript[X, k]), you can be sure that

A.alignerframe[k] = A.(Subscript[X, k]) 
                                           =  ((||A . (Subscript[X, 2])||)/(||A . (Subscript[X, 2])||))  A.(Subscript[X, k])

= ||A.(Subscript[X, k])||  (1/(||A.(Subscript[X, k])||))  A.(Subscript[X, k])

= stretch[k] hangerframe[k]
 because stretch[k] =  ||A.(Subscript[X, k])||  and hangerframe[k] = (1/(||A.(Subscript[X, k])||))  A.(Subscript[X, k]).

▪ Why when you go with
 {alignerframe[1],alignerframe[2],.alignerframe[3], . .,alignerframe[p]}

=  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.

and
         {stretch[1],stretch[2],stretch[3],....,stretch[p]}

= {||A . (Subscript[X, 1])||,||A . (Subscript[X, 2])||,||A . (Subscript[X, 3])||,....,||A . (Subscript[X, p])||}.

and
       {hangerframe[1],hangerframe[2], . . .,hangerframe[p]},

=  {(1/(||A . (Subscript[X, 1])||))A.(Subscript[X, 1]), (1/(||A . (Subscript[X, 2])||))A.(Subscript[X, 2]),(1/(||A . (Subscript[X, 3])||))A.(Subscript[X, 3]),. . .,(1/(||A . (Subscript[X, p])||))A.(Subscript[X, P])}.

you are guaranteed that for any X in hitdimD

A.X =  (∑_(k = 1)^p)stretch[k] (X.alignerframe[k]) hangerframe[k].

The Spectral theorem gives you an orthonormal basis 
           {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} 
of hitdimD consisting of unit eigenvectors of (A^t) . A. Because this is a basis of hitdimD, it spans all of hitdim D, so you can resolve X into its perpendicular components in the directions of these basis vectors.

X = (∑_(k = 1)^hitdim)(X . (Subscript[X, k])) (Subscript[X, k])

Hit both sides with A to get

A.X = (∑_(k = 1)^hitdim)(X . (Subscript[X, k])) A.(Subscript[X, k])

= (∑_(k = 1)^p)(X . (Subscript[X, k])) A.(Subscript[X, k])

= (∑_(k = 1)^p)(X . alignerframe[k]) A.alignerframe[k]

= (∑_(k = 1)^p)(X . alignerframe[k]) stretch[k] hangerframe[k]

= (∑_(k = 1)^p) stretch[k] (X . alignerframe[k]) hangerframe[k]

B.3) The Spectral Theorem as the holy grail of Matrix theory: 
If A is a matrix (symmetric or not symmetric) that hits on hitdimD and if {(Subscript[X, 1]), (Subscript[X, 2]), (Subscript[X, 3]), (Subscript[X, 4]), ...., (Subscript[X, hitdim])} is an orthonormal basis of unit eigenvectors of  (A^t) . A  (as guaranteed by the Spectral Theorem), then you can read off 
-> an orthonormal basis of the column space R[A] of A , 
-> an orthonormal basis of the null space N[A] of A,
-> an orthonormal basis of the row space R[(A^t)] of A,
and
-> the construction of the PseudoInverse of A

In this problem, you get to see how the Spectral Theorem reveals the secrets behind any matrix.
Here are the relevant terms:

Column space R[A] of a matrix A

Given a matrix A, the column space R[A] is the subspace of hangdimD consisting of all possible hits with A.

Row space R[(A^t)] of a matrix A

Given matrix A, the row space R[(A^t)] is the subspace of hitdimD consisting of all possible hits with (A^t).

Null space  N[A] of a matrix A

Given a matrix A, the null space N[A] is the subspace of hitdimD consisting of all X's with A.X = {0,0,...,0}

The pseudo inverse of a matrix A

Given a matrix A, and a vector Y in hangdimD, X = PseudoInverse[A].Y is the vector of smallest norm in hitdimD with the property that A.X is as close to Y as you can get by hitting with A. 
If A is invertible, then PseudoInverse[A] = (A^(-1)).

B.3.a.i)  Using an orthonormal basis{(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])}  unit eigenvectors of  (A^t) . A   to read off 
-> an orthonormal basis of the column space R[A] of A , 
-> an orthonormal basis of the null space N[A] of A,
-> an orthonormal basis of the row space R[(A^t)] of A,
and
-> the construction of the PseudoInverse of A.

Given a matrix A hitting on hitdimD and hanging in hangdim D, you already know that (A^t) . A is a symmetric matrix which hits on hitdimD and hangs in hitdimD.

The Spectral theorem gives you an orthonormal basis {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} of hitdimD consisting of unit eigenvectors of (A^t) . A. Group and (if necessary relabel)  these vectors into two groups.
        
Group I = {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} for which A.(Subscript[X, j]) ≠ {0,0,...0} for all j = 1,2,...,p.

Group 2 = {(Subscript[X, p + 1]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} for which A.(Subscript[X, j]) = {0,0,...} for all j = p+1,p+2, ...,hitdimD.

The facts of the matter are:
▪ The rank of A is p.
▪ To get an orthonormal basis of the column space R[A], you just go with   
             {A . ((Subscript[X, 1])/(||A . (Subscript[X, 1])||)),  
      A . ((Subscript[X, 2])/(||A . (Subscript[X, 2])||)), 
      A . ((Subscript[X, 3])/(||A . (Subscript[X, 3])||)),  ... , 
      A . ((Subscript[X, p])/(||A . (Subscript[X, p])||))}.

▪ To get an orthonormal basis of the row space R[(A^t)], you just go with  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.

▪  To get an orthonormal basis of N[A], you just go with  {(Subscript[X, p + 1]),(Subscript[X, p + 2]),(Subscript[X, 3]),....,(Subscript[X, hitdim])}.

▪  For any Y in hangdimD, you can calculate PseudoInverse[A].Y ( or (A^(-1)) . Y if A is invertible) by going with
             PseudoInverse[A] . Y=\(∑\+\(j = 1\)\%p\( Y• \((A . X\_j)\)\)\/\(\(||\)\(A . X\_j\)\( || \^2\)\)\ X\_j\)

Here's a matrix A.

A  =  
        Table[Random[Real, {-5, 5}], {i, 1, 4}, {j, 1, 6}]; 
MatrixForm[A]

Use the information above to come up with quick calculations of:
▪ The rank of A
▪ An orthonormal basis  the row space of A and the dimension of the row space of A,
▪ An orthonormal basis of the null space of A and the dimension of the null space of A,
▪ An orthonormal basis of the column space of A  and the dimension of the column space of A,
and
▪ PseudoInverse[A].Y for any Y in hangdimD.

Look at A again:

MatrixForm[A]

This matrix hits on 6D and hangs in 4D.

hitdim  =  6
hangdim  =  4

Look at:an orthonormal basis {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])}  of hitdimD consisting of unit eigenvectors of (A^t) . A.

Clear[eigenvector, k];
eigenvector[
          k_]  :=  Eigenvectors[Transpose[A] . A][[k]];
ColumnForm[Table[eigenvector[k], {k, 1, hitdim}]]

See which are hit into {0,0,. . . ,0} by A:

Table[A . eigenvector[k], {k, 1, hitdim}]

The first four are non-zero. 
This signals that the rank of A is 4.:

An orthonormal basis of R[(A^t)], the row space of A is {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4])} ::

p  =  4;
rowspacebasis  =  Table[eigenvector[k], {k, 1, p}]

The dimension of the row space of A is:

p

An orthonormal basis of N[A], the null space of A is {(Subscript[X, 5]), (Subscript[X, 6])} :

nullspacebasis  =  
        Table[eigenvector[k], {k, p + 1, hitdim}];
ColumnForm[nullspacebasis]

The dimension of the null space of A is:

hitdim  -  p

An orthonormal basis {Y[1],Y[2],...,Y[p]} of R[A], the column space of A is:

Clear[Y, k];
Y[k_]  := 
      A . eigenvector[k]/(Sqrt[(A . eigenvector[k]) . (A . eigenvector[k])])
colspacebasis =  Table[Y[k] , {k, 1, p}]

The dimension of the column space of A is:

p

PseudoInverse[A].Y for any Y = {y[1],y[2],...,y[hangdim]} in hangdimD is
  PseudoInverse[A . Y] =\(∑\+\(j = 1\)\%p\( Y• \((A . X\_j)\)\)\/\(\(||\)\(A . X\_j\)\( || \^2\)\)\ X\_j\):

Clear[Y, y];
Y  =  Table[y[k], {k, 1, hangdim}];
Expand[(∑_(k = 1)^p)(((Y . (A . eigenvector[k]) )/((A . eigenvector[k]))) . (A . eigenvector[k])) eigenvector[
            k]]

Check:

PseudoInverse[A] . Y

Nailed it cold.

B.5.a.ii) Explanations

Explain where all this good stuff comes from.

All of it comes from the Spectral theorem applied to a symmetric matrix 
Given a matrix A hitting on hitdimD and hanging in hangdim D, you already know that (A^t) . A is a symmetric matrix which hits on hitdimD and hangs in hitdimD.

The Spectral theorem gives you an orthonormal basis {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} of hitdimD consisting of unit eigenvectors of (A^t) . A. Group and (if necessary relabel)  these vectors into two groups.
        
Group I = {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} for which A.(Subscript[X, j]) ≠ {0,0,...0} for all j = 1,2,...,p.
These are also eigenvectors of (A^t) . A whose corresponding eigenvalues are non -zero.
Group 2 = {(Subscript[X, p + 1]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} for which A.(Subscript[X, j]) = {0,0,...} for all j = p+1,p+2, ...,hitdimD.

To get an orthonormal basis of column space of A, R[A], you just go with  
           {A . ((Subscript[X, 1])/(||A . (Subscript[X, 1])||)),  
      A . ((Subscript[X, 2])/(||A . (Subscript[X, 2])||)), 
      A . ((Subscript[X, 3])/(||A . (Subscript[X, 3])||)),  ... , 
      A . ((Subscript[X, p])/(||A . (Subscript[X, p])||))}.
The rank of A is p.

The Spectral theorem gives you an orthonormal basis {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} of hitdimD consisting of unit eigenvectors of (A^t) . A. 
        
{(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} consists of the unit eigenvectors for which A.(Subscript[X, j]) ≠ {0,0,...} for all j = 1,2,...,p.
Group 1 gives an SVD aligner frame for A. So  
          {A . ((Subscript[X, 1])/(||A . (Subscript[X, 1])||)),  
      A . ((Subscript[X, 2])/(||A . (Subscript[X, 2])||)), 
      A . ((Subscript[X, 3])/(||A . (Subscript[X, 3])||)),  ... , 
      A . ((Subscript[X, p])/(||A . (Subscript[X, p])||))}. 
 is an SVD hanger frame for A. This perpendicular frame automatically spans R[A].
 This also reveals that the rank of A is p.

To get an orthonormal basis of the row space of A,R[(A^t)], you just go with  {(Subscript[X, 1]), (Subscript[X, 2]), (Subscript[X, 3]), ...., (Subscript[X, p])}.
The rank of (A^t) is also p and the dimension of the row space of A is p

Note first:
The (dimension of R[(A^t)]) = ( rank of (A^t))  = (rank of A) = dimension of R[A] = p.
Also note that
 {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} spans a p-dimensional subspace of hitdimD. 
Now, if you can explain why each of the vectors in  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} are also in R[(A^t)] ( the subspace consisting of all hits with (A^t), you will have explained why  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])} is an orthonormal basis of R[(A^t)].
 
To this end, take (Subscript[X, j])  with 1 ≤ j ≤ p and remember that (Subscript[X, j])  is and eigenvector of (A^t) . A with corresponding eigenvalue  (Subscript[λ, j]) .
This tells you (and everyone else) that
           ((A^t) . A) . (Subscript[X, j])  =  (Subscript[λ, j]) (Subscript[X, j ]).
This is the same as saying
            (A^t) . (A . ((Subscript[X, j])/(Subscript[λ, j]))) =  (Subscript[X, j ]).
This tells you that to get  (Subscript[X, j ]), all you have to do its to hit (A . ((Subscript[X, j])/(Subscript[λ, j]))) with (A^t).
This signals that  (Subscript[X, j]) is in R[(A^t)]. 

The upshot: The orthonormal set   {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}  is in R[(A^t)].

Explanation complete.

To get an orthonormal basis of the null space of A, N[A], you just go with  {(Subscript[X, p+1 ]),(Subscript[X, p + 2]),....,(Subscript[X, hitdim])}.
The dimension of the null space of A is hitdim - p

{(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, hitdim])} is an orthonormal basis of hitdimD.
Any X in hitdimD is of this form:
           X = (∑_(j = 1)^hitdim)(X•(Subscript[X, j])) (Subscript[X, j]).
So
           A . X = (∑_(j = 1)^hitdim)(X•(Subscript[X, j])) A . (Subscript[X, j])
                     =(∑_(j = 1)^p)(X•(Subscript[X, j])) (A . (Subscript[X, j]))
because A . (Subscript[X, j])  =  {0, 0,  ... , 0} for j= p+1, p+2, ..., hitdim.

Now remember that 
            (A . (Subscript[X, i])) • (A . (Subscript[X, j]))  =  0 for i ≠ j
and calculate
           (A . X ) • (A . X) =\(∑\+\(j = 1\)\%p\(\(\(||\)\(A . 
              X\_j\)\)\( || \^2\)\ \ \((X•X\_j)\)\^2\)\). 
So saying that A . X = {0, 0,  ... , 0} is the same as saying
             \(∑\+\(j = 1\)\%p\(\(\(||\)\(A . 
              X\_j\)\)\( || \^2\)\ \ \((X•X\_j)\)\^2\)\) = 0.
             
Because each term in the sum is non negative, saying that A . X = {0, 0,  ... , 0}is the same as saying that
                 \(\(\(||\)\(A . X\_j\)\)\( || \^2\)\ \ \((X•X\_j)\)\^2\) = 0 for all j = 1, 2,...p.
                 
 But \(\(\(\(||\)\(A . X\_j\)\)\( || \^2\)\)\) > 0 for each j = 1,2,...p.
 
 So: Saying that  A . X = {0, 0,  ... , 0} is the same as saying that
                  X•(Subscript[X, j]) = 0 for all j = 1, 2,...p.
                 
To recapitulate: 
Starting with
            X = (∑_(j = 1)^hitdim)(X•(Subscript[X, j])) (Subscript[X, j]),
 saying that A . X = {0, 0,  ... , 0} is the same as saying that
            X=(∑_(j = p + 1)^hitdim)(X•(Subscript[X, j])) (Subscript[X, j]) 
 and this is the same as saying that X is in the subspace of hitdimD spanned by
            {(Subscript[X, p + 1]),(Subscript[X, p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])}.
            
This explains why {(Subscript[X, p + 1]),(Subscript[X, 2  p + 2]),(Subscript[X, p + 3]),....,(Subscript[X, hitdim])} is an orthonormal basis of N[A].

For any Y in hangdimD, you can calculate PseudoInverse[A] ( or (A^(-1))if A is invertible) by going with
                    PseudoInverse[A . Y] =\(∑\+\(j = 1\)\%p\( Y• \((A . X\_j)\)\)\/\(\(||\)\(A . X\_j\)\( || \^2\)\)\ X\_j\)

{A . ((Subscript[X, 1])/(||A . (Subscript[X, 1])||)),  
      A . ((Subscript[X, 2])/(||A . (Subscript[X, 2])||)), 
      A . ((Subscript[X, 3])/(||A . (Subscript[X, 3])||)),  ... , 
      A . ((Subscript[X, p])/(||A . (Subscript[X, p])||))} is an orthonormal basis of R[A], the subspace of hangdimD consisting of all possible hits with A. Accordingly if Y is any vector in hangdimD, then the vector in R[A] closest to Y is
           
                R[A]closest=   (∑_(j = 1)^p)(Y•  A . ((Subscript[X, j])/(||A . (Subscript[X, j])||))) ((A.(Subscript[X, j]))/(||A . (Subscript[X, j])||)) =  (∑_(j = 1)^p)\(\(Y• \((A . X\_j)\)\)\/\(\(\(||\)\(A . 
                    X\_j\)\)\( || \^2\)\)\) (A.(Subscript[X, j]))
To get PseudoInverse[A].Y, you just erase the red A to get
                Xapprox  =   \(∑\+\(j = 1\)\%p\( Y• \((A . X\_j)\)\)\/\(\(\(||\)\(A . X\_j\)\)\( || \^2\)\)\ X\_j\) .
Now when you hit Xapprox with A, you get
       A.Xapprox  =  A. \(\(\(∑\+\(j = 1\)\%p\( Y• \((A . X\_j)\)\)\/\(\(\(||\)\(A . X\_j\)\)\( || \^2\)\)\ X\_j\)\(\ \ \)\(=\)\)\)  (∑_(j = 1)^p)\(\(Y• \((A . X\_j)\)\)\/\(\(\(||\)\(A . 
                    X\_j\)\)\( || \^2\)\)\) (A.(Subscript[X, j])) =    R[A]closest.
To see why Xapprox is the vector X in hitdimD of smallest norm satisfying
       A.X =  R[A]closest, 
notice that if X is any other vector with   
           A.X =  R[A]closest,
then X - Xapprox = Z is in N[A]
So 
              X = Xapprox+ Z with Z in N[A]
But:
-> Xapprox is in the span of

{(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}

and 
    Z is the span of

{(Subscript[X, p]),(Subscript[X, p + 2]),....,(Subscript[X, hitdim])}.

This guarantees that
           Xapprox•Z= 0 
 (because all the vectors in the span of {(Subscript[X, p]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, hitdim])} are perpendicular to all vectors in the span of  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]),....,(Subscript[X, p])}.
 So

\(\(\(||\)\(Xapprox + Z\)\( || \^2\)\)\) = (Xapprox + Z) • (Xapprox + Z)  
                                          = Xapprox• Xapprox + 2 ( Xapprox•Z) + Z•Z 
                                          = Xapprox• Xapprox + 2 (0) + Z•Z 
                                          = \(\(\(||\)\(Xapprox\)\( || \^2\)\)\) + \(\(\(||\)\(Z\)\( || \^2\)\)\) > \(\(\(||\)\(Xtest\)\( || \^2\)\)\) 
unless Z = {0,0,...,0}.

B.4) For the record: The transpose manipulation: Y• (A . X) = ((A^t) . Y ) •X

B.3.a) The transpose manipulation: Y• (A . X) = ((A^t) . Y ) •X

The transpose manipulation says that if a matrix A hits on hitdimD and hangs in hangdimD, you can be sure that
           Y• (A . X) = ((A^t) . Y ) •X.
for all hitdimD vectors X and for all hangdimD vectors Y.
Try it out for random vectors X and Y and random matrices A:

hitdim  =  
        Random[Integer, {2, 7}];
hangdim  =  
        Random[Integer, {2, 7}];
Clear[i, j];
A  =  
        Table[Random[Real, {-10, 10}], {i, 1, hangdim}, {j, 1, 
            hitdim}];
MatrixForm[A];
X  =  
        Table[Random[Real, {-10, 10}], {j, 1, hitdim}];
Y  = 
        Table[Random[Real, {-10, 10}], {i, 1, hangdim}];
Y . (A . X)
(Transpose[A] . Y) . X

Although not terribly interesting in its own right, the transpose manipulation 
         Y• (A . X) = ((A^t) . Y ) •X.
lies at the heart of a lot of theory.

For the record, explain why the outcome
          Y• (A . X) = ((A^t) . Y ) •X
is always guaranteed.

Agree that the arbitrary vector X in hitdimD is:
             X = {x[1], x[2], . . ., 
            x[hitdim]}.
Agree that COL[j] is the jth vertical column of A.
Take an arbitrary Y in hangdimD and start bean-counting calculations:
               A.X=(∑_(j = 1)^( hitdim))x[j] COL[j].
So:
              Y.(A.X)=(∑_(j = 1)^hitdim)x[j] (Y . COL[j]).
              
 On the other hand because the columns of A  are the rows of (A^t) ,
               (A^t).Y={(COL[1].Y),(COL[2].Y),. . .,(COL[hitdim].Y)}.
So
              ((A^t).Y).X = (∑_(j = 1)^hitdim) (Y.COL[j])x[j] 
And now because
               (∑_(j = 1)^hitdim) (Y.COL[j])x[j] = (∑_(j = 1)^hitdim)x[j] (Y . COL[j]).,
the inescapable conclusion is
                 Y• (A . X) = ((A^t) . Y ) •X.
Explanation complete.
And you're out of here.

B.5) For the record: An explanation the Spectral Theorem: 
If A is a symmetric kD matrix, then there is a perpendicular frame (orthonormal basis)    
            {(Subscript[X, 1]), (Subscript[X, 2]), (Subscript[X, 3]), (Subscript[X, 4]), ...., (Subscript[X, k])}
of real eigenvectors of A which spans all of kD

B.5.a.i) Visualization

Here's a random symmetric 2D matrix A:

Clear[i, j];
B = 
        Table[Random[Real, {-4, 4}], {i, 1, 2}, {j, 1, 2}];
A = (1/2) (B + Transpose[B]);
MatrixForm[A]

Make this function:

Clear[g, x, y];
\(g[x_, y_] = 
      Expand[{x, y} . \((A . {x, y})\)\/{x, y} . {x, y}]\)

Look at this graphic:

glevels = 
        DensityPlot[Evaluate[g[x, y]], {x, -1, 1}, {y, -1, 1}, 
          Axes -> True, AxesLabel -> {"x", "y"}, 
          PlotPoints -> 50];

Throw in the unit eigenvectors of A:

Clear[eigenvector];
{eigenvector[1], eigenvector[2]} = Eigenvectors[A];
eigenplot = 
        Table[Arrow[eigenvector[k], Tail -> {0, 0}, 
            VectorColor -> Red], {k, 1, 2}];
Show[glevels, eigenplot];

Here's another:

Clear[i, j];
B = 
        Table[Random[Real, {-4, 4}], {i, 1, 2}, {j, 1, 
            2}];
A = 
        (1/2) (B + Transpose[B]);
MatrixForm[
      A]
Clear[g, x, y];
\(g[x_, y_] = 
          Expand[{x, y} . \((A . {x, y})\)\/{x, y} . {x, y}];\)
glevels = 
        DensityPlot[Evaluate[g[x, y]], {x, -1, 1}, {y, -1, 1}, 
          Axes -> True, AxesLabel -> {"x", "y"}, 
          PlotPoints -> 50, 
          DisplayFunction -> Identity];
Clear[
        eigenvector];
{eigenvector[1], eigenvector[2]} = 
        Eigenvectors[A];
eigenplot = 
        Table[Arrow[eigenvector[k], Tail -> {0, 0}, 
            VectorColor -> Red], {k, 1, 2}];
Show[glevels, 
        eigenplot, PlotLabel -> "Density plot with eigenvectors of A", 
        DisplayFunction -> $DisplayFunction];

Rerun a couple of times.
From these graphics, you can see that one eigenvector of the symmetric matrix A points into the place where g[x,y] is biggest and the other eigenvector of A points into the place where g[x,y] is smallest.

What relevance does this have for the explanation of the Spectral Theorem?

These graphics set the strategy for the explanation of the Spectral Theorem.
For more, go on to the next part.

B.5.a.ii) Explanation: The Spectral Theorem for  symmetric matrices

The Spectral Theorem for symmetric matrices says this.
If A is a kD symmetric matrix, then  you are guaranteed a perpendicular frame  {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, k])} of unit eigenvectors of A which spans all of kD .
Explain the reasoning behind the Spectral Theorem.

Step 1: Coming up with an eigenvector (Subscript[X, 1])

Go with a kD symmetric matrix A and put  for each kD vector X
            (Subscript[g, 1])[X] = ((X• (A . X))/X)•X .
            
Notice that
             \(g\_1[
          t\ X] = \(\((t\ X)\) • \((A . \((t\ X)\))\)\)\/\(\((t\ X)\) • \((t\ X)\)\)\) = \(\(\(t\^2\) \((X• \((A . X)\))\)\)\/\(\(t\^2\) \((X•X)\)\)\) = g[X]
so that (Subscript[g, 1])[X] does not depend on the length of X.
The function (Subscript[g, 1])[X] assumes a global maximum value on kD at a unit vector (Subscript[X, 1]).

You are going to see why (Subscript[X, 1])is an eigenvector of A with corresponding eigenvalue (Subscript[X, 1]) • A . (Subscript[X, 1]).

To this end, take any other kD vector Y and put
Put 
        (Subscript[f, 1])[t] = (Subscript[g, 1])[(Subscript[X, 1]) + t Y] 
and notice that t = 0 maximizes (Subscript[f, 1])[t] so that (Subscript[f, 1])'[0] = 0.
      
Now examine (Subscript[f, 1])[t] noticing first that 
          (Subscript[f, 1])[t] = (Subscript[p, 1])[t]/(Subscript[q, 1])[t]
where
            (Subscript[p, 1])[t] = ((Subscript[X, 1]) + t Y) • (A . ((Subscript[X, 1]) + t Y))
                      = (Subscript[X, 1]) • A . (Subscript[X, 1]) + t (Subscript[X, 1]) • (A . Y) + 
        t Y• (A . (Subscript[X, 1])) + (t^2) Y• (A . Y)
                       = (Subscript[X, 1]) • A . (Subscript[X, 1])+t (Subscript[X, 1]) • (A . Y)+t ((A^t) . Y)• (Subscript[X, 1])+(t^2) Y• (A . Y)
                       = (Subscript[X, 1]) • A . (Subscript[X, 1])+t (Subscript[X, 1]) • (A . Y)+t (A . Y)• (Subscript[X, 1])+(t^2) Y• (A . Y)
                        = (Subscript[X, 1]) • A . (Subscript[X, 1]) + 
        2 t Y• (A . (Subscript[X, 1])) + 
        (t^2) Y• (A . Y)
and
              (Subscript[q, 1])[t] =  ((Subscript[X, 1]) + t Y) • ((Subscript[X, 1]) + t Y) 
                         = (Subscript[X, 1]) •(Subscript[X, 1]) + 2 t Y•(Subscript[X, 1]) + 
        (t^2) Y•Y
          
The quotient rule from calculus says
             ((Subscript[f, 1]))'[t]=(((Subscript[q, 1])[t] ((Subscript[p, 1]))'[t]- (Subscript[p, 1])[t] ((Subscript[q, 1]))'[t])/(q[
                t]^2)).
At this stage you know ((Subscript[f, 1]))'[0]=0 so
             (Subscript[q, 1])[0] ((Subscript[p, 1]))'[0]-(Subscript[p, 1])[0] ((Subscript[q, 1]))'[0]=0
             .
Differentiate (Subscript[p, 1])[t] and (Subscript[q, 1])[t] with respect to t to get
       (Subscript[p, 1])′[t] = 2 Y• (A . (Subscript[X, 1])) + 2 t Y• (A . Y)
       ((Subscript[q, 1]))'[t] =  2 Y•(Subscript[X, 1]) + 2 t Y•Y
      
Read off:
      (Subscript[p, 1])[0] = (Subscript[X, 1]) • (A . (Subscript[X, 1])),        ((Subscript[p, 1]))'[0]=2 Y• (A . (Subscript[X, 1]))
      (Subscript[q, 1])[0] = (Subscript[X, 1]) •(Subscript[X, 1 ])             and ((Subscript[q, 1]))'[0]=2 Y• (Subscript[X, 1])
Plug these into
       (Subscript[q, 1])[0] ((Subscript[p, 1]))'[0]-(Subscript[p, 1])[0]((Subscript[q, 1]))'[0]=0 
 to get
       ((Subscript[X, 1]) •(Subscript[X, 1])) (2 Y• (A . (Subscript[X, 1]))) - ((Subscript[X, 1]) • (A . 
                  (Subscript[X, 1]))) (2 Y• (Subscript[X, 1]))   = 0.
 Because (Subscript[X, 1])is a unit vector (so that (Subscript[X, 1]) •(Subscript[X, 1]) = 1) , this is the same as
       (2 Y• (A . (Subscript[X, 1]))) - ((Subscript[X, 1]) • (A . 
                  (Subscript[X, 1]))) (2 Y• (Subscript[X, 1]))   = 0.

 Divide both sides by 2 to get
        (Y• (A . (Subscript[X, 1]))) - ((Subscript[X, 1]) • (A . (Subscript[X, 1]))) ( 
            Y• (Subscript[X, 1]))   = 0.
        
Put
           (Subscript[e, 1]) = (Subscript[X, 1]) • (A . (Subscript[X, 1]))  
to get
        (Y• (A . (Subscript[X, 1]))) - (Subscript[e, 1]) ( Y• (Subscript[X, 1]))   = 0.
        
Factor out Y to get
         Y• ((A . (Subscript[X, 1])) - (Subscript[e, 1])  (Subscript[X, 1]))   =  0.
         
Now remember that Y is any kD vector.  This gives you the freedom to set
         Y =  (A . (Subscript[X, 1])) -  (Subscript[e, 1])  (Subscript[X, 1 ]).
to see that
        \(\(\(\(\(||\)\(A . X\_1\)\) - \ e\_1\ X\_1\)\( || \^2\)\)\) = (A . (Subscript[X, 1]) -  (Subscript[e, 1]) (Subscript[X, 1])) • (A . (Subscript[X, 1]) -  
              (Subscript[e, 1]) (Subscript[X, 1]))  =  0. 
This tells you and everyone else that
          A . (Subscript[X, 1]) -  (Subscript[e, 1]) (Subscript[X, 1])  =  {0, 0, 
          0, . . .0}.
          
This is the same as
          A . (Subscript[X, 1]) =  (Subscript[e, 1]) (Subscript[X, 1 ]),
signalling that (Subscript[X, 1]) is an eigenvector of A with corresponding eigenvalue (Subscript[e, 1]) = (Subscript[X, 1]) • (A . (Subscript[X, 1]))

Step 2: Coming up with a  new eigenvector (Subscript[X, 2]) perpendicular to  (Subscript[X, 1])

Stay with the same kD symmetric matrix A and put (Subscript[S, 2]) = perpendicular complement of the one dimensional subspace of kD spanned by the eigenvector (Subscript[X, 1.]) 
Put  for each kD vector X in  (Subscript[S, 2])
            (Subscript[g, 2])[X] = ((X• (A . X))/X)•X .
            
As in the part above, (Subscript[g, 2])[X] does not depend on the length of X.
The function (Subscript[g, 2])[X] assumes a global maximum value on (Subscript[S, 2]) at a unit vector (Subscript[X, 2]).
You are going to see why (Subscript[X, 2 ])is an eigenvector of A and why (Subscript[X, 2]) is perpendicular to (Subscript[X, 1]).
Things to note:

1) (Subscript[X, 2]) is perpendicular to (Subscript[X, 1]).

Reason:
 (Subscript[X, 2]) is in (Subscript[S, 2]) and everything is (Subscript[S, 2]) is perpendicular to (Subscript[X, 1]).

2) A . (Subscript[X, 2]) is perpendicular to (Subscript[X, 1]) so that A . (Subscript[X, 2]) is in (Subscript[S, 2]).

Reason:
            (Subscript[X, 1]) • (A . (Subscript[X, 2]))= ((A^t) . (Subscript[X, 1])) •(Subscript[X, 2]) = (A . (Subscript[X, 1])) • (Subscript[X, 2]) 
                                   = ((Subscript[e, 1]) . (Subscript[X, 1])) •(Subscript[X, 2]) = (Subscript[e, 1]) ((Subscript[X, 1]) •(Subscript[X, 2]))  =  0
            because (Subscript[X, 2 ]) is in (Subscript[S, 2]) and everything in (Subscript[S, 2]) is perpendicular to (Subscript[X, 1]).

Now for any vector Y in (Subscript[S, 2]), put  
        (Subscript[f, 2])[t] = (Subscript[g, 2])[(Subscript[X, 2]) + t Y] 
and notice that t = 0 maximizes (Subscript[f, 2])[t] so that (Subscript[f, 2])'[0] = 0.

Computations virtually identical to those in the first part reveal that for any Y in (Subscript[S, 2])
     
         Y• ((A . (Subscript[X, 2])) - (Subscript[e, 2])  (Subscript[X, 2]))   =  0.
         where (Subscript[e, 2]) = (Subscript[X, 2]) (A . (Subscript[X, 2])).
         
Now remember that Y is any vector in (Subscript[S, 2]) .  This gives you the freedom to set
         Y =  (A . (Subscript[X, 2])) -  (Subscript[e, 2])  (Subscript[X, 2 ]).
(because both   (A . (Subscript[X, 2])) and (Subscript[e, 2])  (Subscript[X, 2 ]) are in (Subscript[S, 2])). 
to see that
        \(\(\(\(\(||\)\(A . X\_2\)\) - \ e\_2\ X\_2\)\( || \^2\)\)\) = (A . (Subscript[X, 2]) -  (Subscript[e, 2]) (Subscript[X, 2])) • (A . (Subscript[X, 2]) -  
              (Subscript[e, 2]) (Subscript[X, 2]))  =  0. 
This tells you and everyone else that
          A . (Subscript[X, 2]) -  (Subscript[e, 2]) (Subscript[X, 2])  =  {0, 0, 
          0, . . .0}.
          
This is the same as
          A . (Subscript[X, 2]) =  (Subscript[e, 2]) (Subscript[X, 2 ]),
signalling that (Subscript[X, 2]) is an eigenvector of A with corresponding eigenvalue (Subscript[e, 2]) = (Subscript[X, 2]) • (A . (Subscript[X, 2])) 

At this stage, you have two mutually perpendicular eigenvectors (Subscript[X, 1]) and (Subscript[X, 2]) of A.

Step 3: Coming up with a  new eigenvector (Subscript[X, 3]) perpendicular to both  (Subscript[X, 1])and (Subscript[X, 2])

Stay with the same kD symmetric matrix A and put
 (Subscript[S, 3]) = perpendicular complement of the two dimensional subspace of kD spanned by the eigenvectors (Subscript[X, 1])and (Subscript[X, 2]).
Stay with the same kD symmetric matrix A and put  for each kD vector X in  (Subscript[S, 3])
            (Subscript[g, 3])[X] = ((X• (A . X))/X)•X .
            
As in the part above, (Subscript[g, 3])[X] does not depend on the length of X.
The function (Subscript[g, 3])[X] assumes a global maximum value on (Subscript[S, 3]) at a unitvector (Subscript[X, 3]).
You are going to see why (Subscript[X, 3 ])is an eigenvector of A and why (Subscript[X, 3]) is perpendicular to both (Subscript[X, 1])and (Subscript[X, 2]).
Things to note:

1) (Subscript[X, 3]) is perpendicular to both (Subscript[X, 1]) and (Subscript[X, 2])

Reason: (Subscript[X, 3]) is in (Subscript[S, 3]) and everything is (Subscript[S, 3]) is perpendicular to both (Subscript[X, 1]) and (Subscript[X, 2]) .

2) A . (Subscript[X, 3]) is perpendicular to both (Subscript[X, 1]) and (Subscript[X, 2 ])so that A . (Subscript[X, 3]) is in (Subscript[S, 3]).

Reasons:
            (Subscript[X, 1]) • (A . (Subscript[X, 3]))= ((A^t) . (Subscript[X, 1])) •(Subscript[X, 3]) = (A . (Subscript[X, 1])) • (Subscript[X, 3]) 
                                   = ((Subscript[e, 1]) . (Subscript[X, 1])) •(Subscript[X, 3]) = (Subscript[e, 1]) ((Subscript[X, 1]) •(Subscript[X, 3]))  =  0
            because (Subscript[X, 3 ]) is in (Subscript[S, 3]) and everything in (Subscript[S, 3]) is perpendicular to (Subscript[X, 1]). 
            
Also  
            (Subscript[X, 2]) • (A . (Subscript[X, 3]))= ((A^t) . (Subscript[X, 2])) •(Subscript[X, 3]) = (A . (Subscript[X, 2])) • (Subscript[X, 3]) 
                                   = ((Subscript[e, 2]) . (Subscript[X, 2])) •(Subscript[X, 3]) = (Subscript[e, 2]) ((Subscript[X, 2]) •(Subscript[X, 3]))  =  0
            because (Subscript[X, 3 ]) is in (Subscript[S, 3]) and everything in (Subscript[S, 3]) is perpendicular to (Subscript[X, 2]).
This explains why A . (Subscript[X, 3 ])is perpendicular to both  (Subscript[X, 1]) and to (Subscript[X, 2]). And this puts  A . (Subscript[X, 3 ]) into (Subscript[S, 3 ])

Now for any vector Y in (Subscript[S, 3]), put  
        (Subscript[f, 3])[t] = (Subscript[g, 3])[(Subscript[X, 3]) + t Y] 
and notice that t = 0 maximizes (Subscript[f, 3])[t] so that (Subscript[f, 3])'[0] = 0.

Computations virtually identical to those in the first part reveal that for any Y in (Subscript[S, 3])
     
         Y• ((A . (Subscript[X, 3])) - (Subscript[e, 3])  (Subscript[X, 3]))   =  0.
         where (Subscript[e, 3]) = (Subscript[X, 3]) (A . (Subscript[X, 3]))
         
Now remember that Y is any vector in (Subscript[S, 2]) .  This gives you the freedom to set
         Y =  (A . (Subscript[X, 3])) -  (Subscript[e, 3])  (Subscript[X, 3 ]).
(because both   (A . (Subscript[X, 3])) and (Subscript[e, 3])  (Subscript[X, 3 ]) are in (Subscript[S, 3])). 
to see that
        \(\(\(\(\(||\)\(A . X\_3\)\) - \ e\_3\ X\_3\)\( || \^2\)\)\) = (A . (Subscript[X, 3]) -  (Subscript[e, 3]) (Subscript[X, 3])) • (A . (Subscript[X, 3]) -  
              (Subscript[e, 3]) (Subscript[X, 3]))  =  0. 
This tells you and everyone else that
          A . (Subscript[X, 3]) -  (Subscript[e, 3]) (Subscript[X, 3])  =  {0, 0, 
          0, . . .0}.
          
This is the same as
          A . (Subscript[X, 3]) =  (Subscript[e, 3]) (Subscript[X, 3 ]),
signalling that (Subscript[X, 3]) is an eigenvector of A with corresponding eigenvalue (Subscript[e, 3]) = (Subscript[X, 3]) • (A . (Subscript[X, 3])) 

At this stage, you have three mutually perpendicular eigenvectors (Subscript[X, 1 ]) , (Subscript[X, 2 ])and (Subscript[X, 3]) of A.

Step 4 to Step k:

To get the advertised orthonormal basis of eigenvectors of A, you continue the process started above until you run out of dimensions.
This results in a perpendicular frame
           {(Subscript[X, 1]),(Subscript[X, 2]),(Subscript[X, 3]), (Subscript[X, 4]),....,(Subscript[X, k])}
of eigenvectors of A which spans all of kD - an orthonormal basis of kD consisting of eigenvectors of A.