diff --git a/hw7/task2/task2.sln b/hw7/task2/task2.sln new file mode 100644 index 0000000..b5284cc --- /dev/null +++ b/hw7/task2/task2.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.7.34018.315 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "task2", "task2\task2.fsproj", "{7BB5443F-6B84-4C7B-9186-8DA2500AA2A7}" +EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "task2.tests", "task2.tests\task2.tests.fsproj", "{A3267DBC-02BE-4DEC-95C6-040BDE0731F7}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {7BB5443F-6B84-4C7B-9186-8DA2500AA2A7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7BB5443F-6B84-4C7B-9186-8DA2500AA2A7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7BB5443F-6B84-4C7B-9186-8DA2500AA2A7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7BB5443F-6B84-4C7B-9186-8DA2500AA2A7}.Release|Any CPU.Build.0 = Release|Any CPU + {A3267DBC-02BE-4DEC-95C6-040BDE0731F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3267DBC-02BE-4DEC-95C6-040BDE0731F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3267DBC-02BE-4DEC-95C6-040BDE0731F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3267DBC-02BE-4DEC-95C6-040BDE0731F7}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {0A7E6CA8-58E3-4DC0-96B2-D1A33AFCCAE1} + EndGlobalSection +EndGlobal diff --git a/hw7/task2/task2.tests/CrawlerTests.fs b/hw7/task2/task2.tests/CrawlerTests.fs new file mode 100644 index 0000000..c0f3233 --- /dev/null +++ b/hw7/task2/task2.tests/CrawlerTests.fs @@ -0,0 +1,25 @@ +module Crawler.tests + +open NUnit.Framework +open FsUnit +open Crawler +open System + +[] +let SeSiteTest () = + let url = "https://se.math.spbu.ru/practice" + let expected = [ + Some ("https://oops.math.spbu.ru/SE/alumni", 49175) + Some ("https://oops.math.spbu.ru/SE/alumni", 49175) + ] + getPagesInfo url + |> (Async.RunSynchronously) + |> Option.get + |> (Async.RunSynchronously) + |> Seq.toList + |> should equal expected + +[] +let IncorrectLinkTest () = + (fun () -> getPagesInfo "incorrect" |> Async.RunSynchronously |> ignore) + |> should throw typeof diff --git a/hw7/task2/task2.tests/task2.tests.fsproj b/hw7/task2/task2.tests/task2.tests.fsproj new file mode 100644 index 0000000..ceddb78 --- /dev/null +++ b/hw7/task2/task2.tests/task2.tests.fsproj @@ -0,0 +1,28 @@ + + + + net6.0 + + false + true + true + + + + + + + + + + + + + + + + + + + + diff --git a/hw7/task2/task2/Program.fs b/hw7/task2/task2/Program.fs new file mode 100644 index 0000000..4f3e877 --- /dev/null +++ b/hw7/task2/task2/Program.fs @@ -0,0 +1,30 @@ +module Crawler + +open System.Net.Http +open System.Text.RegularExpressions +open System + +let getPagesInfo (url: string) = + async { + try + let linkPattern = Regex("]*>", RegexOptions.Compiled) + let client = new HttpClient() + let! mainPage = (client.GetStringAsync url) |> Async.AwaitTask + + // Gets all matching the pattern links from the main page + let links = mainPage |> linkPattern.Matches |> Seq.map (fun i -> i.Groups[1].Value) + + // Creates a task for each link + let tasks = links |> Seq.map (fun (link: string) -> + async { + try + let! result = Async.AwaitTask(client.GetStringAsync(link)) + return Some(link, result.Length) + with _ -> + return None + }) + return Some(tasks |> Async.Parallel) + with :? Net.WebException -> + raise (InvalidOperationException "Incorrect url") + return None + } diff --git a/hw7/task2/task2/task2.fsproj b/hw7/task2/task2/task2.fsproj new file mode 100644 index 0000000..dd5c71e --- /dev/null +++ b/hw7/task2/task2/task2.fsproj @@ -0,0 +1,12 @@ + + + + Exe + net6.0 + + + + + + +