Crawl a site

Link: https://github.com/jaeles-project/gospider

iron@kali:~/gospider$ ~/go/bin/gospider -s "https://www.mandiant.com/" -o gospider_mandiant                                                                                                                                                                                                                                                                                                             
[url] - [code-200] - https://www.mandiant.com/                                                                                                                                                          
[subdomains] - http://www.mandiant.com                                                                                                                                                                  
[subdomains] - https://www.mandiant.com                                                                                                                                                                 
[subdomains] - http://login.mandiant.com                                                                                                                                                                
[subdomains] - https://login.mandiant.com
[href] - https://selling.mandiant.com/mandiant/sso.aspx                                                                                                                                                 
[href] - https://www.mandiant.com/email-preferences                                                                                                                                                                                                                                                                                                                                                              
[href] - https://www.mandiant.com/customer-success                                                                                                                                                                                                                                                                                                                                                               
[href] - https://policies.google.com/privacy?hl=en-US                                                                                                                                                   
[href] - https://cloud.google.com/product-terms                                                                                                                                                         
[href] - https://www.mandiant.com/compliance                                                                                                                                                            
[href] - https://www.mandiant.com/sitemap.xml                                                                                                                                                                                                                                                                                                                                                                    
[form] - https://www.mandiant.com/                                                                                                                                                                      
[javascript] - https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/google_tag.script.js?rzble3                                                                                   
[javascript] - https://cdn.cookielaw.org/scripttemplates/otSDKStub.js                                                                                                                                   
[javascript] - https://www.mandiant.com/sites/default/files/js/js_2dEw0glALPWbamAn6OQ4XKvUHed5d4MnOAceUi_jNPU.js                                                                                        
[javascript] - https://www.google.com/recaptcha/api.js?onload=CaptchaCallback&render=explicit
[javascript] - https://www.mandiant.com/sites/default/files/js/js_CQg7V4HQPZ_AWJQUTlRAVhsUgo27thNpKFm0x9QBvpI.js                                                                                                                                                                                                                                                                                                 
[javascript] - https://www.mandiant.com/modules/contrib/ckeditor_accordion/js/accordion.frontend.js?rzble3                                                                                              
[javascript] - https://www.mandiant.com/sites/default/files/js/js_EndR9TYA4dXiqr2GB478bpkzEEeSQB0Tuo9UvyeOm-U.js                                                                                        
[url] - [code-200] - https://www.google.com/recaptcha/api.js?onload=CaptchaCallback&render=explicit 
[url] - [code-200] - https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/google_tag.script.js?rzble3                                                                             
[linkfinder] - [from: https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/google_tag.script.js?rzble3] - gtm.js                                                                  
[linkfinder] - https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/gtm.js                                                                                                        
[linkfinder] - https://www.mandiant.com/gtm.js                                                                                                                                                          
[linkfinder] - [from: https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/google_tag.script.js?rzble3] - https://www.googletagmanager.com/gtm.js?id=                             
[linkfinder] - https://www.googletagmanager.com/gtm.js?id=                     
[url] - [code-200] - https://www.mandiant.com/sites/default/files/js/js_EndR9TYA4dXiqr2GB478bpkzEEeSQB0Tuo9UvyeOm-U.js                                                                                  
[url] - [code-200] - https://static.addtoany.com/menu/page.js                                                                                                                                                                                                                                                                                                                                                    
[subdomains] - http://pages.mandiant.com                                                                                                                                                                
[subdomains] - https://pages.mandiant.com

iron@kali:~/gospider$ ls -lsh
total 12K
4.0K drwxrwxr-x 2 iron iron 4.0K Aug 14 12:39 gospider_mandiant
iron@kali:~/gospider$ cd gospider_mandiant/
iron@kali:~/gospider$ ls
www_mandiant_com

iron@kali:~/gospider$ grep subdomains www_mandiant_com 
[subdomains] - https://www.mandiant.com
[subdomains] - https://login.mandiant.com
[subdomains] - https://mwise.mandiant.com
[subdomains] - https://selling.mandiant.com
[subdomains] - https://pages.mandiant.com

iron@kali:~/gospider$:~/gospider/gospider_mandiant$ grep javascript www_mandiant_com 
[javascript] - https://www.mandiant.com/sites/default/files/google_tag/google_tag_manager/google_tag.script.js?rzble3
[javascript] - https://cdn.cookielaw.org/scripttemplates/otSDKStub.js
[javascript] - https://www.mandiant.com/sites/default/files/js/js_2dEw0glALPWbamAn6OQ4XKvUHed5d4MnOAceUi_jNPU.js
[javascript] - https://www.google.com/recaptcha/api.js?onload=CaptchaCallback&render=explicit
[javascript] - https://static.addtoany.com/menu/page.js
[javascript] - https://www.mandiant.com/sites/default/files/js/js_CQg7V4HQPZ_AWJQUTlRAVhsUgo27thNpKFm0x9QBvpI.js
[javascript] - https://www.mandiant.com/modules/contrib/ckeditor_accordion/js/accordion.frontend.js?rzble3
[javascript] - https://www.mandiant.com/sites/default/files/js/js_EndR9TYA4dXiqr2GB478bpkzEEeSQB0Tuo9UvyeOm-U.js
[linkfinder] - [from: https://www.mandiant.com/sites/default/files/js/js_CQg7V4HQPZ_AWJQUTlRAVhsUgo27thNpKFm0x9QBvpI.js] - text/javascript
[linkfinder] - https://www.mandiant.com/sites/default/files/js/text/javascript
[linkfinder] - https://www.mandiant.com/text/javascript
[javascript] - https://cdn.cookielaw.org/scripttemplates/otSDKStub.js
[javascript] - https://www.mandiant.com/sites/default/files/js/js_2dEw0glALPWbamAn6OQ4XKvUHed5d4MnOAceUi_jNPU.js
[javascript] - https://www.google.com/recaptcha/api.js?onload=CaptchaCallback&render=explicit
[javascript] - https://www.mandiant.com/sites/default/files/js/js_CQg7V4HQPZ_AWJQUTlRAVhsUgo27thNpKFm0x9QBvpI.js
[javascript] - https://www.mandiant.com/sites/default/files/js/js_EndR9TYA4dXiqr2GB478bpkzEEeSQB0Tuo9UvyeOm-U.js