From ddb41ec46ddc558be2f43135a1daa01ebac8e841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20P=C3=ADchov=C3=A1?= Date: Mon, 12 Feb 2024 20:40:26 +0000 Subject: [PATCH 001/151] [release/8.0] Update MsQuic --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index c202a5bbb52..e9219d391d6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -219,7 +219,7 @@ 8.0.0-rtm.23523.2 - 2.2.3 + 2.2.5-ci.444313 8.0.0-alpha.1.23527.1 16.0.5-alpha.1.23566.1 From 0c7efec09666babeebea1f608c91757b8faccca0 Mon Sep 17 00:00:00 2001 From: Natalia Kondratyeva Date: Wed, 14 Feb 2024 18:02:10 +0000 Subject: [PATCH 002/151] [release/8.0] Fix HTTP/2 WebSocket Abort --- .../SocketsHttpHandler/Http2Connection.cs | 8 + .../Http/SocketsHttpHandler/Http2Stream.cs | 97 +++++-- ...etsHttpHandlerTest.Http2ExtendedConnect.cs | 157 ++++++++++- .../tests/AbortTest.Loopback.cs | 246 ++++++++++++++++++ .../tests/LoopbackHelper.cs | 21 +- .../LoopbackServer/Http2LoopbackStream.cs | 100 +++++++ .../LoopbackServer/LoopbackWebSocketServer.cs | 148 +++++++++++ .../WebSocketHandshakeHelper.cs | 134 ++++++++++ .../LoopbackServer/WebSocketRequestData.cs | 20 ++ .../System.Net.WebSockets.Client.Tests.csproj | 5 + 10 files changed, 902 insertions(+), 34 deletions(-) create mode 100644 src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs create mode 100644 src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs create mode 100644 src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs create mode 100644 src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs create mode 100644 src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index e9cce1c24d3..14c9685a4f5 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -1446,6 +1446,14 @@ private int WriteHeaderCollection(HttpRequestMessage request, HttpHeaders header continue; } + // Extended connect requests will use the response content stream for bidirectional communication. + // We will ignore any content set for such requests in Http2Stream.SendRequestBodyAsync, as it has no defined semantics. + // Drop the Content-Length header as well in the unlikely case it was set. + if (knownHeader == KnownHeaders.ContentLength && request.IsExtendedConnectRequest) + { + continue; + } + // For all other known headers, send them via their pre-encoded name and the associated value. WriteBytes(knownHeader.Http2EncodedName, ref headerBuffer); string? separator = null; diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs index de66b7cfa10..d834679274b 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs @@ -105,7 +105,9 @@ public Http2Stream(HttpRequestMessage request, Http2Connection connection) _headerBudgetRemaining = connection._pool.Settings.MaxResponseHeadersByteLength; - if (_request.Content == null) + // Extended connect requests will use the response content stream for bidirectional communication. + // We will ignore any content set for such requests in SendRequestBodyAsync, as it has no defined semantics. + if (_request.Content == null || _request.IsExtendedConnectRequest) { _requestCompletionState = StreamCompletionState.Completed; if (_request.IsExtendedConnectRequest) @@ -173,7 +175,9 @@ public HttpResponseMessage GetAndClearResponse() public async Task SendRequestBodyAsync(CancellationToken cancellationToken) { - if (_request.Content == null) + // Extended connect requests will use the response content stream for bidirectional communication. + // Ignore any content set for such requests, as it has no defined semantics. + if (_request.Content == null || _request.IsExtendedConnectRequest) { Debug.Assert(_requestCompletionState == StreamCompletionState.Completed); return; @@ -250,6 +254,7 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) // and we also don't want to propagate any error to the caller, in particular for non-duplex scenarios. Debug.Assert(_responseCompletionState == StreamCompletionState.Completed); _requestCompletionState = StreamCompletionState.Completed; + Debug.Assert(!ConnectProtocolEstablished); Complete(); return; } @@ -261,6 +266,7 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) _requestCompletionState = StreamCompletionState.Failed; SendReset(); + Debug.Assert(!ConnectProtocolEstablished); Complete(); } @@ -313,6 +319,7 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) if (complete) { + Debug.Assert(!ConnectProtocolEstablished); Complete(); } } @@ -420,7 +427,17 @@ private void Cancel() if (sendReset) { SendReset(); - Complete(); + + // Extended CONNECT notes: + // + // To prevent from calling it *twice*, Extended CONNECT stream's Complete() is only + // called from CloseResponseBody(), as CloseResponseBody() is *always* called + // from Extended CONNECT stream's Dispose(). + + if (!ConnectProtocolEstablished) + { + Complete(); + } } } @@ -810,7 +827,20 @@ public void OnHeadersComplete(bool endStream) Debug.Assert(_responseCompletionState == StreamCompletionState.InProgress, $"Response already completed with state={_responseCompletionState}"); _responseCompletionState = StreamCompletionState.Completed; - if (_requestCompletionState == StreamCompletionState.Completed) + + // Extended CONNECT notes: + // + // To prevent from calling it *prematurely*, Extended CONNECT stream's Complete() is only + // called from CloseResponseBody(), as CloseResponseBody() is *only* called + // from Extended CONNECT stream's Dispose(). + // + // Due to bidirectional streaming nature of the Extended CONNECT request, + // the *write side* of the stream can only be completed by calling Dispose(). + // + // The streaming in both ways happens over the single "response" stream instance, which makes + // _requestCompletionState *not indicative* of the actual state of the write side of the stream. + + if (_requestCompletionState == StreamCompletionState.Completed && !ConnectProtocolEstablished) { Complete(); } @@ -871,7 +901,20 @@ public void OnResponseData(ReadOnlySpan buffer, bool endStream) Debug.Assert(_responseCompletionState == StreamCompletionState.InProgress, $"Response already completed with state={_responseCompletionState}"); _responseCompletionState = StreamCompletionState.Completed; - if (_requestCompletionState == StreamCompletionState.Completed) + + // Extended CONNECT notes: + // + // To prevent from calling it *prematurely*, Extended CONNECT stream's Complete() is only + // called from CloseResponseBody(), as CloseResponseBody() is *only* called + // from Extended CONNECT stream's Dispose(). + // + // Due to bidirectional streaming nature of the Extended CONNECT request, + // the *write side* of the stream can only be completed by calling Dispose(). + // + // The streaming in both ways happens over the single "response" stream instance, which makes + // _requestCompletionState *not indicative* of the actual state of the write side of the stream. + + if (_requestCompletionState == StreamCompletionState.Completed && !ConnectProtocolEstablished) { Complete(); } @@ -1036,17 +1079,17 @@ public async Task ReadResponseHeadersAsync(CancellationToken cancellationToken) Debug.Assert(_response != null && _response.Content != null); // Start to process the response body. var responseContent = (HttpConnectionResponseContent)_response.Content; - if (emptyResponse) + if (ConnectProtocolEstablished) + { + responseContent.SetStream(new Http2ReadWriteStream(this, closeResponseBodyOnDispose: true)); + } + else if (emptyResponse) { // If there are any trailers, copy them over to the response. Normally this would be handled by // the response stream hitting EOF, but if there is no response body, we do it here. MoveTrailersToResponseMessage(_response); responseContent.SetStream(EmptyReadStream.Instance); } - else if (ConnectProtocolEstablished) - { - responseContent.SetStream(new Http2ReadWriteStream(this)); - } else { responseContent.SetStream(new Http2ReadStream(this)); @@ -1309,8 +1352,25 @@ private async ValueTask SendDataAsync(ReadOnlyMemory buffer, CancellationT } } + // This method should only be called from Http2ReadWriteStream.Dispose() private void CloseResponseBody() { + // Extended CONNECT notes: + // + // Due to bidirectional streaming nature of the Extended CONNECT request, + // the *write side* of the stream can only be completed by calling Dispose() + // (which, for Extended CONNECT case, will in turn call CloseResponseBody()) + // + // Similarly to QuicStream, disposal *gracefully* closes the write side of the stream + // (unless we've received RST_STREAM before) and *abortively* closes the read side + // of the stream (unless we've received EOS before). + + if (ConnectProtocolEstablished && _resetException is null) + { + // Gracefully close the write side of the Extended CONNECT stream + _connection.LogExceptions(_connection.SendEndStreamAsync(StreamId)); + } + // Check if the response body has been fully consumed. bool fullyConsumed = false; Debug.Assert(!Monitor.IsEntered(SyncObject)); @@ -1323,6 +1383,7 @@ private void CloseResponseBody() } // If the response body isn't completed, cancel it now. + // This includes aborting the read side of the Extended CONNECT stream. if (!fullyConsumed) { Cancel(); @@ -1337,6 +1398,12 @@ private void CloseResponseBody() lock (SyncObject) { + if (ConnectProtocolEstablished) + { + // This should be the only place where Extended Connect stream is completed + Complete(); + } + _responseBuffer.Dispose(); } } @@ -1430,10 +1497,7 @@ private enum StreamCompletionState : byte private sealed class Http2ReadStream : Http2ReadWriteStream { - public Http2ReadStream(Http2Stream http2Stream) : base(http2Stream) - { - base.CloseResponseBodyOnDispose = true; - } + public Http2ReadStream(Http2Stream http2Stream) : base(http2Stream, closeResponseBodyOnDispose: true) { } public override bool CanWrite => false; @@ -1482,12 +1546,13 @@ public class Http2ReadWriteStream : HttpBaseStream private Http2Stream? _http2Stream; private readonly HttpResponseMessage _responseMessage; - public Http2ReadWriteStream(Http2Stream http2Stream) + public Http2ReadWriteStream(Http2Stream http2Stream, bool closeResponseBodyOnDispose = false) { Debug.Assert(http2Stream != null); Debug.Assert(http2Stream._response != null); _http2Stream = http2Stream; _responseMessage = _http2Stream._response; + CloseResponseBodyOnDispose = closeResponseBodyOnDispose; } ~Http2ReadWriteStream() @@ -1503,7 +1568,7 @@ public Http2ReadWriteStream(Http2Stream http2Stream) } } - protected bool CloseResponseBodyOnDispose { get; set; } + protected bool CloseResponseBodyOnDispose { get; private init; } protected override void Dispose(bool disposing) { diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs index 0ea6ae9e13f..cb1a15df14e 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs @@ -2,8 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Net.Test.Common; +using System.Threading; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -31,6 +33,7 @@ public static IEnumerable UseSsl_MemberData() [MemberData(nameof(UseSsl_MemberData))] public async Task Connect_ReadWriteResponseStream(bool useSsl) { + const int MessageCount = 3; byte[] clientMessage = new byte[] { 1, 2, 3 }; byte[] serverMessage = new byte[] { 4, 5, 6, 7 }; @@ -43,19 +46,39 @@ await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync(async uri HttpRequestMessage request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); request.Headers.Protocol = "foo"; + bool readFromContentStream = false; + + // We won't send the content bytes, but we will send content headers. + // Since we're dropping the content, we'll also drop the Content-Length header. + request.Content = new StreamContent(new DelegateStream( + readAsyncFunc: (_, _, _, _) => + { + readFromContentStream = true; + throw new UnreachableException(); + })); + + request.Headers.Add("User-Agent", "foo"); + request.Content.Headers.Add("Content-Language", "bar"); + request.Content.Headers.ContentLength = 42; + using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); using Stream responseStream = await response.Content.ReadAsStreamAsync(); - await responseStream.WriteAsync(clientMessage); - await responseStream.FlushAsync(); + for (int i = 0; i < MessageCount; i++) + { + await responseStream.WriteAsync(clientMessage); + await responseStream.FlushAsync(); - byte[] readBuffer = new byte[serverMessage.Length]; - await responseStream.ReadExactlyAsync(readBuffer); - Assert.Equal(serverMessage, readBuffer); + byte[] readBuffer = new byte[serverMessage.Length]; + await responseStream.ReadExactlyAsync(readBuffer); + Assert.Equal(serverMessage, readBuffer); + } // Receive server's EOS - Assert.Equal(0, await responseStream.ReadAsync(readBuffer)); + Assert.Equal(0, await responseStream.ReadAsync(new byte[1])); + + Assert.False(readFromContentStream); clientCompleted.SetResult(); }, @@ -63,14 +86,21 @@ await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync(async uri { await using Http2LoopbackConnection connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync(new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); - (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); + (int streamId, HttpRequestData request) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); + + Assert.Equal("foo", request.GetSingleHeaderValue("User-Agent")); + Assert.Equal("bar", request.GetSingleHeaderValue("Content-Language")); + Assert.Equal(0, request.GetHeaderValueCount("Content-Length")); await connection.SendResponseHeadersAsync(streamId, endStream: false).ConfigureAwait(false); - DataFrame dataFrame = await connection.ReadDataFrameAsync(); - Assert.Equal(clientMessage, dataFrame.Data.ToArray()); + for (int i = 0; i < MessageCount; i++) + { + DataFrame dataFrame = await connection.ReadDataFrameAsync(); + Assert.Equal(clientMessage, dataFrame.Data.ToArray()); - await connection.SendResponseDataAsync(streamId, serverMessage, endStream: true); + await connection.SendResponseDataAsync(streamId, serverMessage, endStream: i == MessageCount - 1); + } await clientCompleted.Task.WaitAsync(TestHelper.PassingTestTimeout); }, options: new GenericLoopbackOptions { UseSsl = useSsl }); @@ -163,5 +193,112 @@ await server.AcceptConnectionAsync(async connection => await new[] { serverTask, clientTask }.WhenAllOrAnyFailed().WaitAsync(TestHelper.PassingTestTimeout); } + + [Theory] + [MemberData(nameof(UseSsl_MemberData))] + public async Task Connect_ServerSideEOS_ReceivedByClient(bool useSsl) + { + var timeoutTcs = new CancellationTokenSource(TestHelper.PassingTestTimeout); + var serverReceivedEOS = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync( + clientFunc: async uri => + { + var client = CreateHttpClient(); + var request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); + request.Headers.Protocol = "foo"; + + var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutTcs.Token); + var responseStream = await response.Content.ReadAsStreamAsync(timeoutTcs.Token); + + // receive server's EOS + Assert.Equal(0, await responseStream.ReadAsync(new byte[1], timeoutTcs.Token)); + + // send client's EOS + responseStream.Dispose(); + + // wait for "ack" from server + await serverReceivedEOS.Task.WaitAsync(timeoutTcs.Token); + + // can dispose handler now + client.Dispose(); + }, + serverFunc: async server => + { + await using var connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync( + new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); + + (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); + await connection.SendResponseHeadersAsync(streamId, endStream: false); + + // send server's EOS + await connection.SendResponseDataAsync(streamId, Array.Empty(), endStream: true); + + // receive client's EOS "in response" to server's EOS + var eosFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); + Assert.Equal(streamId, eosFrame.StreamId); + Assert.Equal(0, eosFrame.Data.Length); + Assert.True(eosFrame.EndStreamFlag); + + serverReceivedEOS.SetResult(); + + // on handler dispose, client should shutdown the connection without sending additional frames + await connection.WaitForClientDisconnectAsync().WaitAsync(timeoutTcs.Token); + }, + options: new GenericLoopbackOptions { UseSsl = useSsl }); + } + + [Theory] + [MemberData(nameof(UseSsl_MemberData))] + public async Task Connect_ClientSideEOS_ReceivedByServer(bool useSsl) + { + var timeoutTcs = new CancellationTokenSource(TestHelper.PassingTestTimeout); + var serverReceivedRst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync( + clientFunc: async uri => + { + var client = CreateHttpClient(); + var request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); + request.Headers.Protocol = "foo"; + + var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutTcs.Token); + var responseStream = await response.Content.ReadAsStreamAsync(timeoutTcs.Token); + + // send client's EOS + // this will also send RST_STREAM as we didn't receive server's EOS before + responseStream.Dispose(); + + // wait for "ack" from server + await serverReceivedRst.Task.WaitAsync(timeoutTcs.Token); + + // can dispose handler now + client.Dispose(); + }, + serverFunc: async server => + { + await using var connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync( + new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); + + (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); + await connection.SendResponseHeadersAsync(streamId, endStream: false); + + // receive client's EOS + var eosFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); + Assert.Equal(streamId, eosFrame.StreamId); + Assert.Equal(0, eosFrame.Data.Length); + Assert.True(eosFrame.EndStreamFlag); + + // receive client's RST_STREAM as we didn't send server's EOS before + var rstFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); + Assert.Equal(streamId, rstFrame.StreamId); + + serverReceivedRst.SetResult(); + + // on handler dispose, client should shutdown the connection without sending additional frames + await connection.WaitForClientDisconnectAsync().WaitAsync(timeoutTcs.Token); + }, + options: new GenericLoopbackOptions { UseSsl = useSsl }); + } } } diff --git a/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs b/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs new file mode 100644 index 00000000000..0aa83697a9d --- /dev/null +++ b/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs @@ -0,0 +1,246 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace System.Net.WebSockets.Client.Tests +{ + [ConditionalClass(typeof(ClientWebSocketTestBase), nameof(WebSocketsSupported))] + [SkipOnPlatform(TestPlatforms.Browser, "System.Net.Sockets are not supported on browser")] + public abstract class AbortTest_Loopback : ClientWebSocketTestBase + { + public AbortTest_Loopback(ITestOutputHelper output) : base(output) { } + + protected virtual Version HttpVersion => Net.HttpVersion.Version11; + + [Theory] + [MemberData(nameof(AbortClient_MemberData))] + public Task AbortClient_ServerGetsCorrectException(AbortType abortType, bool useSsl, bool verifySendReceive) + { + var clientMsg = new byte[] { 1, 2, 3, 4, 5, 6 }; + var serverMsg = new byte[] { 42 }; + var clientAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var serverAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var timeoutCts = new CancellationTokenSource(TimeOutMilliseconds); + + return LoopbackWebSocketServer.RunAsync( + async (clientWebSocket, token) => + { + if (verifySendReceive) + { + await VerifySendReceiveAsync(clientWebSocket, clientMsg, serverMsg, clientAckTcs, serverAckTcs.Task, token); + } + + switch (abortType) + { + case AbortType.Abort: + clientWebSocket.Abort(); + break; + case AbortType.Dispose: + clientWebSocket.Dispose(); + break; + } + }, + async (serverWebSocket, token) => + { + if (verifySendReceive) + { + await VerifySendReceiveAsync(serverWebSocket, serverMsg, clientMsg, serverAckTcs, clientAckTcs.Task, token); + } + + var readBuffer = new byte[1]; + var exception = await Assert.ThrowsAsync(async () => + await serverWebSocket.ReceiveAsync(readBuffer, token)); + + Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); + Assert.Equal(WebSocketState.Aborted, serverWebSocket.State); + }, + new LoopbackWebSocketServer.Options(HttpVersion, useSsl, GetInvoker()), + timeoutCts.Token); + } + + [Theory] + [MemberData(nameof(ServerPrematureEos_MemberData))] + public Task ServerPrematureEos_ClientGetsCorrectException(ServerEosType serverEosType, bool useSsl) + { + var clientMsg = new byte[] { 1, 2, 3, 4, 5, 6 }; + var serverMsg = new byte[] { 42 }; + var clientAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var serverAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + var timeoutCts = new CancellationTokenSource(TimeOutMilliseconds); + + var globalOptions = new LoopbackWebSocketServer.Options(HttpVersion, useSsl, HttpInvoker: null) + { + DisposeServerWebSocket = false, + ManualServerHandshakeResponse = true + }; + + var serverReceivedEosTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var clientReceivedEosTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + return LoopbackWebSocketServer.RunAsync( + async uri => + { + var token = timeoutCts.Token; + var clientOptions = globalOptions with { HttpInvoker = GetInvoker() }; + var clientWebSocket = await LoopbackWebSocketServer.GetConnectedClientAsync(uri, clientOptions, token).ConfigureAwait(false); + + if (serverEosType == ServerEosType.AfterSomeData) + { + await VerifySendReceiveAsync(clientWebSocket, clientMsg, serverMsg, clientAckTcs, serverAckTcs.Task, token).ConfigureAwait(false); + } + + // only one side of the stream was closed. the other should work + await clientWebSocket.SendAsync(clientMsg, WebSocketMessageType.Binary, endOfMessage: true, token).ConfigureAwait(false); + + var exception = await Assert.ThrowsAsync(() => clientWebSocket.ReceiveAsync(new byte[1], token)); + Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); + + clientReceivedEosTcs.SetResult(); + clientWebSocket.Dispose(); + }, + async (requestData, token) => + { + WebSocket serverWebSocket = null!; + await SendServerResponseAndEosAsync( + requestData, + serverEosType, + (wsData, ct) => + { + var wsOptions = new WebSocketCreationOptions { IsServer = true }; + serverWebSocket = WebSocket.CreateFromStream(wsData.WebSocketStream, wsOptions); + + return serverEosType == ServerEosType.AfterSomeData + ? VerifySendReceiveAsync(serverWebSocket, serverMsg, clientMsg, serverAckTcs, clientAckTcs.Task, ct) + : Task.CompletedTask; + }, + token); + + Assert.NotNull(serverWebSocket); + + // only one side of the stream was closed. the other should work + var readBuffer = new byte[clientMsg.Length]; + var result = await serverWebSocket.ReceiveAsync(readBuffer, token); + Assert.Equal(WebSocketMessageType.Binary, result.MessageType); + Assert.Equal(clientMsg.Length, result.Count); + Assert.True(result.EndOfMessage); + Assert.Equal(clientMsg, readBuffer); + + await clientReceivedEosTcs.Task.WaitAsync(token).ConfigureAwait(false); + + var exception = await Assert.ThrowsAsync(() => serverWebSocket.ReceiveAsync(readBuffer, token)); + Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); + + serverWebSocket.Dispose(); + }, + globalOptions, + timeoutCts.Token); + } + + protected virtual Task SendServerResponseAndEosAsync(WebSocketRequestData requestData, ServerEosType serverEosType, Func serverFunc, CancellationToken cancellationToken) + => WebSocketHandshakeHelper.SendHttp11ServerResponseAndEosAsync(requestData, serverFunc, cancellationToken); // override for HTTP/2 + + private static readonly bool[] Bool_Values = new[] { false, true }; + private static readonly bool[] UseSsl_Values = PlatformDetection.SupportsAlpn ? Bool_Values : new[] { false }; + + public static IEnumerable AbortClient_MemberData() + { + foreach (var abortType in Enum.GetValues()) + { + foreach (var useSsl in UseSsl_Values) + { + foreach (var verifySendReceive in Bool_Values) + { + yield return new object[] { abortType, useSsl, verifySendReceive }; + } + } + } + } + + public static IEnumerable ServerPrematureEos_MemberData() + { + foreach (var serverEosType in Enum.GetValues()) + { + foreach (var useSsl in UseSsl_Values) + { + yield return new object[] { serverEosType, useSsl }; + } + } + } + + public enum AbortType + { + Abort, + Dispose + } + + public enum ServerEosType + { + WithHeaders, + RightAfterHeaders, + AfterSomeData + } + + private static async Task VerifySendReceiveAsync(WebSocket ws, byte[] localMsg, byte[] remoteMsg, + TaskCompletionSource localAckTcs, Task remoteAck, CancellationToken cancellationToken) + { + var sendTask = ws.SendAsync(localMsg, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken); + + var recvBuf = new byte[remoteMsg.Length * 2]; + var recvResult = await ws.ReceiveAsync(recvBuf, cancellationToken).ConfigureAwait(false); + + Assert.Equal(WebSocketMessageType.Binary, recvResult.MessageType); + Assert.Equal(remoteMsg.Length, recvResult.Count); + Assert.True(recvResult.EndOfMessage); + Assert.Equal(remoteMsg, recvBuf[..recvResult.Count]); + + localAckTcs.SetResult(); + + await sendTask.ConfigureAwait(false); + await remoteAck.WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + // --- HTTP/1.1 WebSocket loopback tests --- + + public class AbortTest_Invoker_Loopback : AbortTest_Loopback + { + public AbortTest_Invoker_Loopback(ITestOutputHelper output) : base(output) { } + protected override bool UseCustomInvoker => true; + } + + public class AbortTest_HttpClient_Loopback : AbortTest_Loopback + { + public AbortTest_HttpClient_Loopback(ITestOutputHelper output) : base(output) { } + protected override bool UseHttpClient => true; + } + + public class AbortTest_SharedHandler_Loopback : AbortTest_Loopback + { + public AbortTest_SharedHandler_Loopback(ITestOutputHelper output) : base(output) { } + } + + // --- HTTP/2 WebSocket loopback tests --- + + public class AbortTest_Invoker_Http2 : AbortTest_Invoker_Loopback + { + public AbortTest_Invoker_Http2(ITestOutputHelper output) : base(output) { } + protected override Version HttpVersion => Net.HttpVersion.Version20; + protected override Task SendServerResponseAndEosAsync(WebSocketRequestData rd, ServerEosType eos, Func callback, CancellationToken ct) + => WebSocketHandshakeHelper.SendHttp2ServerResponseAndEosAsync(rd, eosInHeadersFrame: eos == ServerEosType.WithHeaders, callback, ct); + } + + public class AbortTest_HttpClient_Http2 : AbortTest_HttpClient_Loopback + { + public AbortTest_HttpClient_Http2(ITestOutputHelper output) : base(output) { } + protected override Version HttpVersion => Net.HttpVersion.Version20; + protected override Task SendServerResponseAndEosAsync(WebSocketRequestData rd, ServerEosType eos, Func callback, CancellationToken ct) + => WebSocketHandshakeHelper.SendHttp2ServerResponseAndEosAsync(rd, eosInHeadersFrame: eos == ServerEosType.WithHeaders, callback, ct); + } +} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs index 48d167b072f..cee509ee068 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs @@ -28,14 +28,7 @@ public static async Task> WebSocketHandshakeAsync(Loo if (headerName == "Sec-WebSocket-Key") { string headerValue = tokens[1].Trim(); - string responseSecurityAcceptValue = ComputeWebSocketHandshakeSecurityAcceptValue(headerValue); - serverResponse = - "HTTP/1.1 101 Switching Protocols\r\n" + - "Content-Length: 0\r\n" + - "Upgrade: websocket\r\n" + - "Connection: Upgrade\r\n" + - (extensions is null ? null : $"Sec-WebSocket-Extensions: {extensions}\r\n") + - "Sec-WebSocket-Accept: " + responseSecurityAcceptValue + "\r\n\r\n"; + serverResponse = GetServerResponseString(headerValue, extensions); } } } @@ -50,6 +43,18 @@ public static async Task> WebSocketHandshakeAsync(Loo return null; } + public static string GetServerResponseString(string secWebSocketKey, string? extensions = null) + { + var responseSecurityAcceptValue = ComputeWebSocketHandshakeSecurityAcceptValue(secWebSocketKey); + return + "HTTP/1.1 101 Switching Protocols\r\n" + + "Content-Length: 0\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + (extensions is null ? null : $"Sec-WebSocket-Extensions: {extensions}\r\n") + + "Sec-WebSocket-Accept: " + responseSecurityAcceptValue + "\r\n\r\n"; + } + private static string ComputeWebSocketHandshakeSecurityAcceptValue(string secWebSocketKey) { // GUID specified by RFC 6455. diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs new file mode 100644 index 00000000000..1b3b51840ec --- /dev/null +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Net.Test.Common +{ + public class Http2LoopbackStream : Stream + { + private readonly Http2LoopbackConnection _connection; + private readonly int _streamId; + private bool _readEnded; + private ReadOnlyMemory _leftoverReadData; + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => true; + + public Http2LoopbackConnection Connection => _connection; + public int StreamId => _streamId; + + public Http2LoopbackStream(Http2LoopbackConnection connection, int streamId) + { + _connection = connection; + _streamId = streamId; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + if (!_leftoverReadData.IsEmpty) + { + int read = Math.Min(buffer.Length, _leftoverReadData.Length); + _leftoverReadData.Span.Slice(0, read).CopyTo(buffer.Span); + _leftoverReadData = _leftoverReadData.Slice(read); + return read; + } + + if (_readEnded) + { + return 0; + } + + DataFrame dataFrame = (DataFrame)await _connection.ReadFrameAsync(cancellationToken); + Assert.Equal(_streamId, dataFrame.StreamId); + _leftoverReadData = dataFrame.Data; + _readEnded = dataFrame.EndStreamFlag; + + return await ReadAsync(buffer, cancellationToken); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + await _connection.SendResponseDataAsync(_streamId, buffer, endStream: false); + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + + protected override void Dispose(bool disposing) => DisposeAsync().GetAwaiter().GetResult(); + + public override async ValueTask DisposeAsync() + { + try + { + await _connection.SendResponseDataAsync(_streamId, Memory.Empty, endStream: true).ConfigureAwait(false); + + if (!_readEnded) + { + var rstFrame = new RstStreamFrame(FrameFlags.None, (int)ProtocolErrors.NO_ERROR, _streamId); + await _connection.WriteFrameAsync(rstFrame).ConfigureAwait(false); + } + } + catch (IOException) + { + // Ignore connection errors + } + catch (SocketException) + { + // Ignore connection errors + } + } + + public override void Flush() { } + public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); + public override void SetLength(long value) => throw new NotImplementedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + public override long Length => throw new NotImplementedException(); + public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + } +} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs new file mode 100644 index 00000000000..b24e2e20d40 --- /dev/null +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs @@ -0,0 +1,148 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Net.Http; +using System.Net.Test.Common; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Net.WebSockets.Client.Tests +{ + public static class LoopbackWebSocketServer + { + public static Task RunAsync( + Func clientWebSocketFunc, + Func serverWebSocketFunc, + Options options, + CancellationToken cancellationToken) + { + Assert.False(options.ManualServerHandshakeResponse, "Not supported in this overload"); + + return RunAsyncPrivate( + uri => RunClientAsync(uri, clientWebSocketFunc, options, cancellationToken), + (requestData, token) => RunServerAsync(requestData, serverWebSocketFunc, options, token), + options, + cancellationToken); + } + + public static Task RunAsync( + Func loopbackClientFunc, + Func loopbackServerFunc, + Options options, + CancellationToken cancellationToken) + { + Assert.False(options.DisposeClientWebSocket, "Not supported in this overload"); + Assert.False(options.DisposeServerWebSocket, "Not supported in this overload"); + Assert.False(options.DisposeHttpInvoker, "Not supported in this overload"); + Assert.Null(options.HttpInvoker); // Not supported in this overload + + return RunAsyncPrivate(loopbackClientFunc, loopbackServerFunc, options, cancellationToken); + } + + private static Task RunAsyncPrivate( + Func loopbackClientFunc, + Func loopbackServerFunc, + Options options, + CancellationToken cancellationToken) + { + bool sendDefaultServerHandshakeResponse = !options.ManualServerHandshakeResponse; + if (options.HttpVersion == HttpVersion.Version11) + { + return LoopbackServer.CreateClientAndServerAsync( + loopbackClientFunc, + async server => + { + await server.AcceptConnectionAsync(async connection => + { + var requestData = await WebSocketHandshakeHelper.ProcessHttp11RequestAsync(connection, sendDefaultServerHandshakeResponse, cancellationToken).ConfigureAwait(false); + await loopbackServerFunc(requestData, cancellationToken).ConfigureAwait(false); + }); + }, + new LoopbackServer.Options { WebSocketEndpoint = true, UseSsl = options.UseSsl }); + } + else if (options.HttpVersion == HttpVersion.Version20) + { + return Http2LoopbackServer.CreateClientAndServerAsync( + loopbackClientFunc, + async server => + { + var requestData = await WebSocketHandshakeHelper.ProcessHttp2RequestAsync(server, sendDefaultServerHandshakeResponse, cancellationToken).ConfigureAwait(false); + var http2Connection = requestData.Http2Connection!; + var http2StreamId = requestData.Http2StreamId.Value; + + await loopbackServerFunc(requestData, cancellationToken).ConfigureAwait(false); + + await http2Connection.DisposeAsync().ConfigureAwait(false); + }, + new Http2Options { WebSocketEndpoint = true, UseSsl = options.UseSsl }); + } + else + { + throw new ArgumentException(nameof(options.HttpVersion)); + } + } + + private static async Task RunServerAsync( + WebSocketRequestData requestData, + Func serverWebSocketFunc, + Options options, + CancellationToken cancellationToken) + { + var wsOptions = new WebSocketCreationOptions { IsServer = true }; + var serverWebSocket = WebSocket.CreateFromStream(requestData.WebSocketStream, wsOptions); + + await serverWebSocketFunc(serverWebSocket, cancellationToken).ConfigureAwait(false); + + if (options.DisposeServerWebSocket) + { + serverWebSocket.Dispose(); + } + } + + private static async Task RunClientAsync( + Uri uri, + Func clientWebSocketFunc, + Options options, + CancellationToken cancellationToken) + { + var clientWebSocket = await GetConnectedClientAsync(uri, options, cancellationToken).ConfigureAwait(false); + + await clientWebSocketFunc(clientWebSocket, cancellationToken).ConfigureAwait(false); + + if (options.DisposeClientWebSocket) + { + clientWebSocket.Dispose(); + } + + if (options.DisposeHttpInvoker) + { + options.HttpInvoker?.Dispose(); + } + } + + public static async Task GetConnectedClientAsync(Uri uri, Options options, CancellationToken cancellationToken) + { + var clientWebSocket = new ClientWebSocket(); + clientWebSocket.Options.HttpVersion = options.HttpVersion; + clientWebSocket.Options.HttpVersionPolicy = HttpVersionPolicy.RequestVersionExact; + + if (options.UseSsl && options.HttpInvoker is null) + { + clientWebSocket.Options.RemoteCertificateValidationCallback = delegate { return true; }; + } + + await clientWebSocket.ConnectAsync(uri, options.HttpInvoker, cancellationToken).ConfigureAwait(false); + + return clientWebSocket; + } + + public record class Options(Version HttpVersion, bool UseSsl, HttpMessageInvoker? HttpInvoker) + { + public bool DisposeServerWebSocket { get; set; } = true; + public bool DisposeClientWebSocket { get; set; } + public bool DisposeHttpInvoker { get; set; } + public bool ManualServerHandshakeResponse { get; set; } + } + } +} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs new file mode 100644 index 00000000000..f4d2f42f5ed --- /dev/null +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Sockets; +using System.Net.Test.Common; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Net.WebSockets.Client.Tests +{ + public static class WebSocketHandshakeHelper + { + public static async Task ProcessHttp11RequestAsync(LoopbackServer.Connection connection, bool sendServerResponse = true, CancellationToken cancellationToken = default) + { + List headers = await connection.ReadRequestHeaderAsync().WaitAsync(cancellationToken).ConfigureAwait(false); + + var data = new WebSocketRequestData() + { + HttpVersion = HttpVersion.Version11, + Http11Connection = connection + }; + + foreach (string header in headers.Skip(1)) + { + string[] tokens = header.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); + if (tokens.Length is 1 or 2) + { + data.Headers.Add( + tokens[0].Trim(), + tokens.Length == 2 ? tokens[1].Trim() : null); + } + } + + var isValidOpeningHandshake = data.Headers.TryGetValue("Sec-WebSocket-Key", out var secWebSocketKey); + Assert.True(isValidOpeningHandshake); + + if (sendServerResponse) + { + await SendHttp11ServerResponseAsync(connection, secWebSocketKey, cancellationToken).ConfigureAwait(false); + } + + data.WebSocketStream = connection.Stream; + return data; + } + + private static async Task SendHttp11ServerResponseAsync(LoopbackServer.Connection connection, string secWebSocketKey, CancellationToken cancellationToken) + { + var serverResponse = LoopbackHelper.GetServerResponseString(secWebSocketKey); + await connection.WriteStringAsync(serverResponse).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + public static async Task ProcessHttp2RequestAsync(Http2LoopbackServer server, bool sendServerResponse = true, CancellationToken cancellationToken = default) + { + var connection = await server.EstablishConnectionAsync(new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }) + .WaitAsync(cancellationToken).ConfigureAwait(false); + + (int streamId, var httpRequestData) = await connection.ReadAndParseRequestHeaderAsync(readBody: false) + .WaitAsync(cancellationToken).ConfigureAwait(false); + + var data = new WebSocketRequestData + { + HttpVersion = HttpVersion.Version20, + Http2Connection = connection, + Http2StreamId = streamId + }; + + foreach (var header in httpRequestData.Headers) + { + Assert.NotNull(header.Name); + data.Headers.Add(header.Name, header.Value); + } + + var isValidOpeningHandshake = httpRequestData.Method == HttpMethod.Connect.ToString() && data.Headers.ContainsKey(":protocol"); + Assert.True(isValidOpeningHandshake); + + if (sendServerResponse) + { + await SendHttp2ServerResponseAsync(connection, streamId, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + data.WebSocketStream = new Http2LoopbackStream(connection, streamId); + return data; + } + + private static async Task SendHttp2ServerResponseAsync(Http2LoopbackConnection connection, int streamId, bool endStream = false, CancellationToken cancellationToken = default) + { + // send status 200 OK to establish websocket + // we don't need to send anything additional as Sec-WebSocket-Key is not used for HTTP/2 + // note: endStream=true is abnormal and used for testing premature EOS scenarios only + await connection.SendResponseHeadersAsync(streamId, endStream: endStream).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + public static async Task SendHttp11ServerResponseAndEosAsync(WebSocketRequestData requestData, Func? requestDataCallback, CancellationToken cancellationToken) + { + Assert.Equal(HttpVersion.Version11, requestData.HttpVersion); + + // sending default handshake response + await SendHttp11ServerResponseAsync(requestData.Http11Connection!, requestData.Headers["Sec-WebSocket-Key"], cancellationToken).ConfigureAwait(false); + + if (requestDataCallback is not null) + { + await requestDataCallback(requestData, cancellationToken).ConfigureAwait(false); + } + + // send server EOS (half-closing from server side) + requestData.Http11Connection!.Socket.Shutdown(SocketShutdown.Send); + } + + public static async Task SendHttp2ServerResponseAndEosAsync(WebSocketRequestData requestData, bool eosInHeadersFrame, Func? requestDataCallback, CancellationToken cancellationToken) + { + Assert.Equal(HttpVersion.Version20, requestData.HttpVersion); + + var connection = requestData.Http2Connection!; + var streamId = requestData.Http2StreamId!.Value; + + await SendHttp2ServerResponseAsync(connection, streamId, endStream: eosInHeadersFrame, cancellationToken).ConfigureAwait(false); + + if (requestDataCallback is not null) + { + await requestDataCallback(requestData, cancellationToken).ConfigureAwait(false); + } + + if (!eosInHeadersFrame) + { + // send server EOS (half-closing from server side) + await connection.SendResponseDataAsync(streamId, Array.Empty(), endStream: true).ConfigureAwait(false); + } + } + } +} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs new file mode 100644 index 00000000000..799157a370f --- /dev/null +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs @@ -0,0 +1,20 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.IO; +using System.Net.Test.Common; + +namespace System.Net.WebSockets.Client.Tests +{ + public class WebSocketRequestData + { + public Dictionary Headers { get; set; } = new Dictionary(); + public Stream? WebSocketStream { get; set; } + + public Version HttpVersion { get; set; } + public LoopbackServer.Connection? Http11Connection { get; set; } + public Http2LoopbackConnection? Http2Connection { get; set; } + public int? Http2StreamId { get; set; } + } +} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj index 8f23e7925a4..a4f20d03002 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj +++ b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj @@ -55,6 +55,7 @@ + @@ -64,6 +65,10 @@ + + + + From 91b2946fbcfcd7f6cd6c3ce65a6146efb7081173 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:56:30 -0800 Subject: [PATCH 003/151] [release/8.0-staging] Manually depad RSAES-PKCS1 on Apple OSes Co-authored-by: Jeremy Barton --- .../Interop.RSA.cs | 95 +++++++++++++-- .../Cryptography/RsaPaddingProcessor.cs | 104 ++++++++++++++++ .../RSA/EncryptDecrypt.cs | 115 ++++++++++++++++++ .../entrypoints.c | 1 + .../pal_rsa.c | 7 ++ .../pal_rsa.h | 8 ++ 6 files changed, 319 insertions(+), 11 deletions(-) diff --git a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.RSA.cs b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.RSA.cs index 4a3bd4454ed..deab51eeb2b 100644 --- a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.RSA.cs +++ b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.RSA.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Diagnostics; using System.Runtime.InteropServices; using System.Security.Cryptography; @@ -69,8 +70,8 @@ private static partial int RsaDecryptOaep( out SafeCFDataHandle pEncryptedOut, out SafeCFErrorHandle pErrorOut); - [LibraryImport(Libraries.AppleCryptoNative, EntryPoint = "AppleCryptoNative_RsaDecryptPkcs")] - private static partial int RsaDecryptPkcs( + [LibraryImport(Libraries.AppleCryptoNative, EntryPoint = "AppleCryptoNative_RsaDecryptRaw")] + private static partial int RsaDecryptRaw( SafeSecKeyRefHandle publicKey, ReadOnlySpan pbData, int cbData, @@ -166,17 +167,40 @@ internal static byte[] RsaDecrypt( byte[] data, RSAEncryptionPadding padding) { + if (padding == RSAEncryptionPadding.Pkcs1) + { + byte[] padded = ExecuteTransform( + data, + (ReadOnlySpan source, out SafeCFDataHandle decrypted, out SafeCFErrorHandle error) => + RsaDecryptRaw(privateKey, source, source.Length, out decrypted, out error)); + + byte[] depad = CryptoPool.Rent(padded.Length); + OperationStatus status = RsaPaddingProcessor.DepadPkcs1Encryption(padded, depad, out int written); + byte[]? ret = null; + + if (status == OperationStatus.Done) + { + ret = depad.AsSpan(0, written).ToArray(); + } + + // Clear the whole thing, especially on failure. + CryptoPool.Return(depad); + CryptographicOperations.ZeroMemory(padded); + + if (ret is null) + { + throw new CryptographicException(SR.Cryptography_InvalidPadding); + } + + return ret; + } + + Debug.Assert(padding.Mode == RSAEncryptionPaddingMode.Oaep); + return ExecuteTransform( data, (ReadOnlySpan source, out SafeCFDataHandle decrypted, out SafeCFErrorHandle error) => { - if (padding == RSAEncryptionPadding.Pkcs1) - { - return RsaDecryptPkcs(privateKey, source, source.Length, out decrypted, out error); - } - - Debug.Assert(padding.Mode == RSAEncryptionPaddingMode.Oaep); - return RsaDecryptOaep( privateKey, source, @@ -195,14 +219,63 @@ internal static bool TryRsaDecrypt( out int bytesWritten) { Debug.Assert(padding.Mode == RSAEncryptionPaddingMode.Pkcs1 || padding.Mode == RSAEncryptionPaddingMode.Oaep); + + if (padding.Mode == RSAEncryptionPaddingMode.Pkcs1) + { + byte[] padded = CryptoPool.Rent(source.Length); + byte[] depad = CryptoPool.Rent(source.Length); + + bool processed = TryExecuteTransform( + source, + padded, + out int paddedLength, + (ReadOnlySpan innerSource, out SafeCFDataHandle outputHandle, out SafeCFErrorHandle errorHandle) => + RsaDecryptRaw(privateKey, innerSource, innerSource.Length, out outputHandle, out errorHandle)); + + Debug.Assert( + processed, + "TryExecuteTransform should always return true for a large enough buffer."); + + OperationStatus status = OperationStatus.InvalidData; + int depaddedLength = 0; + + if (processed) + { + status = RsaPaddingProcessor.DepadPkcs1Encryption( + new ReadOnlySpan(padded, 0, paddedLength), + depad, + out depaddedLength); + } + + CryptoPool.Return(padded); + + if (status == OperationStatus.Done) + { + if (depaddedLength <= destination.Length) + { + depad.AsSpan(0, depaddedLength).CopyTo(destination); + CryptoPool.Return(depad); + bytesWritten = depaddedLength; + return true; + } + + CryptoPool.Return(depad); + bytesWritten = 0; + return false; + } + + CryptoPool.Return(depad); + Debug.Assert(status == OperationStatus.InvalidData); + throw new CryptographicException(SR.Cryptography_InvalidPadding); + } + return TryExecuteTransform( source, destination, out bytesWritten, delegate (ReadOnlySpan innerSource, out SafeCFDataHandle outputHandle, out SafeCFErrorHandle errorHandle) { - return padding.Mode == RSAEncryptionPaddingMode.Pkcs1 ? - RsaDecryptPkcs(privateKey, innerSource, innerSource.Length, out outputHandle, out errorHandle) : + return RsaDecryptOaep(privateKey, innerSource, innerSource.Length, PalAlgorithmFromAlgorithmName(padding.OaepHashAlgorithm), out outputHandle, out errorHandle); }); } diff --git a/src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs b/src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs index 66256440b7e..a9f190617fb 100644 --- a/src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs +++ b/src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Buffers.Binary; using System.Collections.Concurrent; using System.Diagnostics; @@ -142,6 +143,109 @@ internal static void PadPkcs1Encryption( source.CopyTo(mInEM); } + internal static OperationStatus DepadPkcs1Encryption( + ReadOnlySpan source, + Span destination, + out int bytesWritten) + { + int primitive = DepadPkcs1Encryption(source); + int primitiveSign = SignStretch(primitive); + + // Primitive is a positive length, or ~length to indicate + // an error, so flip ~length to length if the high bit is set. + int len = Choose(primitiveSign, ~primitive, primitive); + int spaceRemain = destination.Length - len; + int spaceRemainSign = SignStretch(spaceRemain); + + // len = clampHigh(len, destination.Length); + len = Choose(spaceRemainSign, destination.Length, len); + + // ret = spaceRemain < 0 ? DestinationTooSmall : Done + int ret = Choose( + spaceRemainSign, + (int)OperationStatus.DestinationTooSmall, + (int)OperationStatus.Done); + + // ret = primitive < 0 ? InvalidData : ret; + ret = Choose(primitiveSign, (int)OperationStatus.InvalidData, ret); + + // Write some number of bytes, regardless of the final return. + source[^len..].CopyTo(destination); + + // bytesWritten = ret == Done ? len : 0; + bytesWritten = Choose(CheckZero(ret), len, 0); + return (OperationStatus)ret; + } + + private static int DepadPkcs1Encryption(ReadOnlySpan source) + { + Debug.Assert(source.Length > 11); + ReadOnlySpan afterPadding = source.Slice(10); + ReadOnlySpan noZeros = source.Slice(2, 8); + + // Find the first zero in noZeros, or -1 for no zeros. + int zeroPos = BlindFindFirstZero(noZeros); + + // If zeroPos is negative, valid is -1, otherwise 0. + int valid = SignStretch(zeroPos); + + // If there are no zeros in afterPadding then zeroPos is negative, + // so negating the sign stretch is 0, which makes hasPos 0. + // If there -was- a zero, sign stretching is 0, so negating it makes hasPos -1. + zeroPos = BlindFindFirstZero(afterPadding); + int hasLen = ~SignStretch(zeroPos); + valid &= hasLen; + + // Check that the first two bytes are { 00 02 } + valid &= CheckZero(source[0] | (source[1] ^ 0x02)); + + int lenIfGood = afterPadding.Length - zeroPos - 1; + // If there were no zeros, use the full after-min-padding segment. + int lenIfBad = ~Choose(hasLen, lenIfGood, source.Length - 11); + + Debug.Assert(lenIfBad < 0); + return Choose(valid, lenIfGood, lenIfBad); + } + + private static int BlindFindFirstZero(ReadOnlySpan source) + { + // Any vectorization of this routine needs to use non-early termination, + // and instructions that do not vary their completion time on the input. + + int pos = -1; + + for (int i = source.Length - 1; i >= 0; i--) + { + // pos = source[i] == 0 ? i : pos; + int local = CheckZero(source[i]); + pos = Choose(local, i, pos); + } + + return pos; + } + + private static int SignStretch(int value) + { + return value >> 31; + } + + private static int Choose(int selector, int yes, int no) + { + Debug.Assert((selector | (selector - 1)) == -1); + return (selector & yes) | (~selector & no); + } + + private static int CheckZero(int value) + { + // For zero, ~value and value-1 are both all bits set (negative). + // For positive values, ~value is negative and value-1 is positive. + // For negative values except MinValue, ~value is positive and value-1 is negative. + // For MinValue, ~value is positive and value-1 is also positive. + // All together, the only thing that has negative & negative is 0, so stretch the sign bit. + int mask = ~value & (value - 1); + return SignStretch(mask); + } + internal static void PadPkcs1Signature( HashAlgorithmName hashAlgorithmName, ReadOnlySpan source, diff --git a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs index 39f3ebc82ec..5778b66d043 100644 --- a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs +++ b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; using Test.Cryptography; using Microsoft.DotNet.XUnitExtensions; using Xunit; @@ -736,6 +738,119 @@ public void Decrypt_Pkcs1_ErrorsForInvalidPadding(byte[] data) } } + [Fact] + public void Decrypt_Pkcs1_BadPadding() + { + if ((PlatformDetection.IsWindows && !PlatformDetection.IsWindows10Version2004OrGreater)) + { + return; + } + + RSAParameters keyParams = TestData.RSA2048Params; + BigInteger e = new BigInteger(keyParams.Exponent, true, true); + BigInteger n = new BigInteger(keyParams.Modulus, true, true); + byte[] buf = new byte[keyParams.Modulus.Length]; + byte[] c = new byte[buf.Length]; + + buf[1] = 2; + buf.AsSpan(2).Fill(1); + + ref byte afterMinPadding = ref buf[10]; + ref byte lastByte = ref buf[^1]; + afterMinPadding = 0; + + using (RSA rsa = RSAFactory.Create(keyParams)) + { + RawEncrypt(buf, e, n, c); + // Assert.NoThrow, check that manual padding is coherent + Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1); + + // All RSA encryption schemes start with 00, so pick any other number. + // + // If buf > modulus then encrypt should fail, so this + // is the largest legal-but-invalid value to test. + buf[0] = keyParams.Modulus[0]; + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // Check again with a zero length payload + (afterMinPadding, lastByte) = (lastByte, afterMinPadding); + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // Back to valid padding + buf[0] = 0; + (afterMinPadding, lastByte) = (lastByte, afterMinPadding); + RawEncrypt(buf, e, n, c); + Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1); + + // This is (sort of) legal for PKCS1 signatures, but not decryption. + buf[1] = 1; + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // No RSA PKCS1 padding scheme starts with 00 FF. + buf[1] = 255; + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // Check again with a zero length payload + (afterMinPadding, lastByte) = (lastByte, afterMinPadding); + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // Back to valid padding + buf[1] = 2; + (afterMinPadding, lastByte) = (lastByte, afterMinPadding); + RawEncrypt(buf, e, n, c); + Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1); + + // Try a zero in every possible required padding position + for (int i = 2; i < 10; i++) + { + buf[i] = 0; + + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // It used to be 1, now it's 2, still not zero. + buf[i] = 2; + } + + // Back to valid padding + RawEncrypt(buf, e, n, c); + Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1); + + // Make it such that + // "there is no octet with hexadecimal value 0x00 to separate PS from M" + // (RFC 3447 sec 7.2.2, rule 3, third clause) + buf.AsSpan(10).Fill(3); + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + + // Every possible problem, for good measure. + buf[0] = 2; + buf[1] = 0; + buf[4] = 0; + RawEncrypt(buf, e, n, c); + Assert.ThrowsAny(() => Decrypt(rsa, c, RSAEncryptionPadding.Pkcs1)); + } + + static void RawEncrypt(ReadOnlySpan source, BigInteger e, BigInteger n, Span destination) + { + BigInteger m = new BigInteger(source, true, true); + BigInteger c = BigInteger.ModPow(m, e, n); + int shift = destination.Length - c.GetByteCount(true); + destination.Slice(0, shift).Clear(); + bool wrote = c.TryWriteBytes(destination.Slice(shift), out int written, true, true); + + if (!wrote || written + shift != destination.Length) + { + throw new UnreachableException(); + } + } + } + public static IEnumerable OaepPaddingModes { get diff --git a/src/native/libs/System.Security.Cryptography.Native.Apple/entrypoints.c b/src/native/libs/System.Security.Cryptography.Native.Apple/entrypoints.c index 9f91b6d2488..099fb343947 100644 --- a/src/native/libs/System.Security.Cryptography.Native.Apple/entrypoints.c +++ b/src/native/libs/System.Security.Cryptography.Native.Apple/entrypoints.c @@ -70,6 +70,7 @@ static const Entry s_cryptoAppleNative[] = DllImportEntry(AppleCryptoNative_RsaGenerateKey) DllImportEntry(AppleCryptoNative_RsaDecryptOaep) DllImportEntry(AppleCryptoNative_RsaDecryptPkcs) + DllImportEntry(AppleCryptoNative_RsaDecryptRaw) DllImportEntry(AppleCryptoNative_RsaEncryptOaep) DllImportEntry(AppleCryptoNative_RsaEncryptPkcs) DllImportEntry(AppleCryptoNative_RsaSignaturePrimitive) diff --git a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.c b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.c index a9aece35fb0..1746828d5b0 100644 --- a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.c +++ b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.c @@ -134,6 +134,13 @@ int32_t AppleCryptoNative_RsaDecryptOaep(SecKeyRef privateKey, privateKey, pbData, cbData, pDecryptedOut, pErrorOut, mgfAlgorithm, SecKeyCreateDecryptedData); } +int32_t AppleCryptoNative_RsaDecryptRaw( + SecKeyRef privateKey, uint8_t* pbData, int32_t cbData, CFDataRef* pDecryptedOut, CFErrorRef* pErrorOut) +{ + return RsaPrimitive( + privateKey, pbData, cbData, pDecryptedOut, pErrorOut, kSecKeyAlgorithmRSAEncryptionRaw, SecKeyCreateDecryptedData); +} + int32_t AppleCryptoNative_RsaDecryptPkcs( SecKeyRef privateKey, uint8_t* pbData, int32_t cbData, CFDataRef* pDecryptedOut, CFErrorRef* pErrorOut) { diff --git a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.h b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.h index 253fdae78e4..34a350f80f9 100644 --- a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.h +++ b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_rsa.h @@ -31,6 +31,14 @@ PALEXPORT int32_t AppleCryptoNative_RsaDecryptOaep(SecKeyRef privateKey, CFDataRef* pDecryptedOut, CFErrorRef* pErrorOut); +/* +Decrypt the contents of pbData using the provided privateKey without validating or removing padding. + +Follows pal_seckey return conventions. +*/ +PALEXPORT int32_t AppleCryptoNative_RsaDecryptRaw( + SecKeyRef privateKey, uint8_t* pbData, int32_t cbData, CFDataRef* pDecryptedOut, CFErrorRef* pErrorOut); + /* Decrypt the contents of pbData using the provided privateKey under PKCS#1 padding. From fe2ea4ef2b70e1399de2036d443fcb26695aa549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20K=C3=B6plinger?= Date: Tue, 20 Feb 2024 10:20:44 +0100 Subject: [PATCH 004/151] Fix NativeAOT publish failure on fi_FI culture (#98552) (#98601) This culture uses `U+2212 : MINUS SIGN` instead of `-` for negative numbers which trips up msbuild when comparing the property. Instead of using an intermediate property just inline the usage and use `Contains()` for better readability. Fixes https://github.com/dotnet/runtime/issues/98550 (cherry picked from commit c768315c5698391f45a6a47d56fec4ba3df59fb8) --- .../Microsoft.DotNet.ILCompiler.SingleEntry.targets | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets index 178d8aaa93a..299840345a7 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets @@ -5,8 +5,7 @@ <_hostOS>$(NETCoreSdkPortableRuntimeIdentifier.SubString(0, $(NETCoreSdkPortableRuntimeIdentifier.LastIndexOf('-')))) <_targetOS>$(RuntimeIdentifier.SubString(0, $(RuntimeIdentifier.LastIndexOf('-')))) - <_indexOfPeriod>$(_targetOS.IndexOf('.')) - <_targetOS Condition="'$(_indexOfPeriod)' > -1">$(_targetOS.SubString(0, $(_indexOfPeriod))) + <_targetOS Condition="$(_targetOS.Contains('.'))">$(_targetOS.SubString(0, $(_targetOS.IndexOf('.')))) <_targetOS Condition="$(_targetOS.StartsWith('win'))">win From 2ebbc777a1142b948a06c6d77a0400a71336124c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 19:18:25 -0600 Subject: [PATCH 005/151] [release/8.0-staging] Update dependencies from dotnet/runtime-assets (#98560) * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240215.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24108.4 -> To Version 8.0.0-beta.24115.2 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 6 +---- eng/Version.Details.xml | 56 ++++++++++++++++++++--------------------- eng/Versions.props | 28 ++++++++++----------- 3 files changed, 43 insertions(+), 47 deletions(-) diff --git a/NuGet.config b/NuGet.config index 2bb901348e5..80d8b321e96 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,12 +9,8 @@ - - - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4e522812e23..06c3925d623 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -185,57 +185,57 @@ https://github.com/dotnet/arcade 61ae141d2bf3534619265c8f691fd55dc3e75147 - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed https://github.com/dotnet/llvm-project @@ -358,9 +358,9 @@ https://github.com/dotnet/hotreload-utils bc857c64c5c5f1fc73048261e8f471c3310224d2 - + https://github.com/dotnet/runtime-assets - ca6c46012f68934198ce0d303196c3ae179230f5 + 0827f89642a6443c96a87990e3b910f6aa4cc4ed https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 4ca58c3abe6..10419d98ece 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,20 +143,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 - 8.0.0-beta.24108.4 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 + 8.0.0-beta.24115.2 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 From c3a4e1383aea019a73d75cc6866bf47af116cba9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 19:20:16 -0600 Subject: [PATCH 006/151] Update dependencies from https://github.com/dotnet/arcade build 20240213.2 (#98445) Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.24059.4 -> To Version 8.0.0-beta.24113.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 72 +++++++++---------- eng/Versions.props | 30 ++++---- eng/common/post-build/publish-using-darc.ps1 | 4 +- .../templates/job/publish-build-assets.yml | 14 ++-- .../templates/post-build/post-build.yml | 16 ++--- .../templates/variables/pool-providers.yml | 12 ++-- global.json | 6 +- 7 files changed, 77 insertions(+), 77 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 06c3925d623..4f40ec3d700 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -111,9 +111,9 @@ - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 @@ -121,69 +121,69 @@ 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 https://github.com/dotnet/runtime-assets @@ -334,9 +334,9 @@ https://github.com/dotnet/xharness a417169d3ba558fd6daa522f04e686574bbce520 - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index 10419d98ece..47287d03ea8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,21 +87,21 @@ 8.0.100 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 2.5.1-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 - 8.0.0-beta.24059.4 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 2.5.1-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 + 8.0.0-beta.24113.2 6.0.0-preview.1.102 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1e779fec4dd..5a3a32ea8d7 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index fa5446c093d..8ec0151def2 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -58,7 +58,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -71,7 +71,7 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 - task: PowerShell@2 @@ -86,7 +86,7 @@ jobs: /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -95,7 +95,7 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: @@ -121,7 +121,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') + condition: eq(variables['SymbolExclusionFile'], 'true') inputs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container @@ -137,7 +137,7 @@ jobs: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -148,4 +148,4 @@ jobs: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f74abf7ce0..aba44a25a33 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,7 +39,7 @@ parameters: displayName: Enable NuGet validation type: boolean default: true - + - name: publishInstallersAndChecksums displayName: Publish installers and checksums type: boolean @@ -131,8 +131,8 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - job: displayName: Signing Validation @@ -221,9 +221,9 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) continueOnError: true @@ -258,7 +258,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -272,7 +272,7 @@ stages: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3b..d236f9fdbb1 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,15 +1,15 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. -# Motivation: +# Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing # (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. -# How to use: +# How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). # If we find alternate naming conventions in broad usage it can be added to the condition below. # @@ -54,4 +54,4 @@ variables: False, 'NetCore1ESPool-Internal' ) - ] \ No newline at end of file + ] diff --git a/global.json b/global.json index 13f92203054..c987cd4bc00 100644 --- a/global.json +++ b/global.json @@ -8,9 +8,9 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24059.4", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24059.4", - "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24059.4", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24113.2", + "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24113.2", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "8.0.0-rc.1.23406.6" From b41d2b68fe5d4c21d8bc1a340343a8015a060175 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 19:25:29 -0600 Subject: [PATCH 007/151] Update dependencies from https://github.com/dotnet/xharness build 20240212.2 (#98444) Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Common , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 8.0.0-prerelease.24060.1 -> To Version 8.0.0-prerelease.24112.2 Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 72af997c0c2..6ff1559a00a 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "8.0.0-prerelease.24060.1", + "version": "8.0.0-prerelease.24112.2", "commands": [ "xharness" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4f40ec3d700..2427ba2b42b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -322,17 +322,17 @@ https://github.com/dotnet/runtime edbd5c769a19798b6955050baccf99e6797d3208 - + https://github.com/dotnet/xharness - a417169d3ba558fd6daa522f04e686574bbce520 + c055cc57f21796e79ace4bca2b070a8777f2446a - + https://github.com/dotnet/xharness - a417169d3ba558fd6daa522f04e686574bbce520 + c055cc57f21796e79ace4bca2b070a8777f2446a - + https://github.com/dotnet/xharness - a417169d3ba558fd6daa522f04e686574bbce520 + c055cc57f21796e79ace4bca2b070a8777f2446a https://github.com/dotnet/arcade diff --git a/eng/Versions.props b/eng/Versions.props index 47287d03ea8..b760f86c84c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -183,9 +183,9 @@ 1.1.0 17.4.0-preview-20220707-01 - 8.0.0-prerelease.24060.1 - 8.0.0-prerelease.24060.1 - 8.0.0-prerelease.24060.1 + 8.0.0-prerelease.24112.2 + 8.0.0-prerelease.24112.2 + 8.0.0-prerelease.24112.2 8.0.0-alpha.0.24072.2 2.4.2 1.0.0 From a3de6b7fbb8cc20637318187ca5b293692cfa76d Mon Sep 17 00:00:00 2001 From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:03:50 -0800 Subject: [PATCH 008/151] Update diasymreader to 17.8.7-beta1.24113.1 (#98539) --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 0c986b60330..fd241deaae3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -165,7 +165,7 @@ 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 - 16.11.29-beta1.23404.4 + 17.8.7-beta1.24113.1 2.0.0-beta4.23307.1 3.0.3 2.1.0 From 276022237eb57cd632bb4a205520a90308799d4f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:39:19 +0100 Subject: [PATCH 009/151] [mono] Set /DEBUGTYPE:CV,FIXUP on binaries (#99356) This fixes an issue with running APIScan on mono-aot-cross.exe --- src/mono/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mono/CMakeLists.txt b/src/mono/CMakeLists.txt index 05766210ea6..8320e4c4cf1 100644 --- a/src/mono/CMakeLists.txt +++ b/src/mono/CMakeLists.txt @@ -284,6 +284,7 @@ elseif(CLR_CMAKE_HOST_OS STREQUAL "windows") add_compile_options(/GL) # whole program optimization add_link_options(/LTCG) # link-time code generation add_link_options(/DEBUG) # enable debugging information + add_link_options(/DEBUGTYPE:CV,FIXUP) # enable fixup debug information add_link_options(/OPT:REF) # optimize: remove unreferenced functions & data add_link_options(/OPT:ICF) # optimize: enable COMDAT folding # the combination of /Zi compiler flag and /DEBUG /OPT:REF /OPT:ICF From 13d578bab9c53546a87920e55255f09e2d75274f Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:06:12 -0800 Subject: [PATCH 010/151] Update branding to 8.0.4 (#99324) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 00d310742cb..5b91c0c7996 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 8.0.3 + 8.0.4 8 0 - 3 + 4 8.0.100 7.0.$([MSBuild]::Add($(PatchVersion),14)) 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) From e398dd64ac7c2a40eefb476f9e76cd7c0e9c5ddb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:29:40 -0800 Subject: [PATCH 011/151] Restore erroneously removed encoding of the argument count in a generic method instantiation (#98825) Users of the multicore jit feature would encounter silent encoding failures while recording profiles due to a defect in our encoding for generic method instances that would make them impossible to decode later. This PR fixes the encoding so that newly-recorded profiles will not be corrupt. Co-authored-by: Katelyn Gadd --- src/coreclr/vm/zapsig.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/coreclr/vm/zapsig.cpp b/src/coreclr/vm/zapsig.cpp index bbbab9a51c8..dfd447f94ea 100644 --- a/src/coreclr/vm/zapsig.cpp +++ b/src/coreclr/vm/zapsig.cpp @@ -1350,6 +1350,9 @@ BOOL ZapSig::EncodeMethod( else { Instantiation inst = pMethod->GetMethodInstantiation(); + + pSigBuilder->AppendData(inst.GetNumArgs()); + for (DWORD i = 0; i < inst.GetNumArgs(); i++) { TypeHandle t = inst[i]; From d66d7908f99087b194f6540317c43faff725b4f8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 11:34:55 -0800 Subject: [PATCH 012/151] [release/8.0-staging] Update dependencies from dotnet/emsdk (#98458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update dependencies from https://github.com/dotnet/emsdk build 20240214.4 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.3-servicing.24108.3 -> To Version 8.0.3-servicing.24114.4 * Update dependencies from https://github.com/dotnet/emsdk build 20240214.6 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.3-servicing.24108.3 -> To Version 8.0.3-servicing.24114.6 * Update dependencies from https://github.com/dotnet/emsdk build 20240215.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.3-servicing.24108.3 -> To Version 8.0.3-servicing.24115.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240227.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.3-servicing.24108.3 -> To Version 8.0.3-servicing.24127.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240306.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.3-servicing.24108.3 -> To Version 8.0.4-servicing.24156.2 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Larry Ewing Co-authored-by: Carlos Sánchez López <1175054+carlossanlop@users.noreply.github.com> --- NuGet.config | 3 +-- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/NuGet.config b/NuGet.config index 80d8b321e96..97380116615 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,8 +9,7 @@ - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2427ba2b42b..44ca79b371d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 9a29abdd764a4de0f253ed368871877a47725247 + 1639670c6547454278f51afc5c74e20f8acc7abd - + https://github.com/dotnet/emsdk - 9a29abdd764a4de0f253ed368871877a47725247 + 1639670c6547454278f51afc5c74e20f8acc7abd diff --git a/eng/Versions.props b/eng/Versions.props index fd241deaae3..b158b3a828d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -240,7 +240,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.3 + 8.0.4 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From 4a1e5f4db0f5c338e2090144ea2cc439bcd865df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 15:02:52 -0500 Subject: [PATCH 013/151] [release/8.0-staging] Fix FormatQuantiles formatting in MetricsEventSource (#99045) * Fix FormatQuantiles formatting in MetricsEventSource These doubles need to be formatted with the invariant culture to meet consumer expectations around parsing them. * Enable DiagnosticSource for servicing --------- Co-authored-by: Stephen Toub Co-authored-by: Eric StJohn --- ...System.Diagnostics.DiagnosticSource.csproj | 2 + .../Diagnostics/Metrics/MetricsEventSource.cs | 6 +- .../tests/MetricEventSourceTests.cs | 119 ++++++++++-------- 3 files changed, 73 insertions(+), 54 deletions(-) diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj b/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj index 49b7834a9de..37825100656 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj @@ -6,6 +6,8 @@ $(NoWarn);SA1205 false true + true + 1 Provides Classes that allow you to decouple code logging rich (unserializable) diagnostics/telemetry (e.g. framework) from code that consumes it (e.g. tools) Commonly Used Types: diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs index 41979421991..4b5da60a441 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs @@ -737,7 +737,11 @@ private static string FormatQuantiles(QuantileValue[] quantiles) StringBuilder sb = new StringBuilder(); for (int i = 0; i < quantiles.Length; i++) { - sb.Append(quantiles[i].Quantile).Append('=').Append(quantiles[i].Value); +#if NETCOREAPP + sb.Append(CultureInfo.InvariantCulture, $"{quantiles[i].Quantile}={quantiles[i].Value}"); +#else + sb.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", quantiles[i].Quantile, quantiles[i].Value); +#endif if (i != quantiles.Length - 1) { sb.Append(';'); diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs b/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs index 1902c2a6c4b..a6d2c4a6f26 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Globalization; @@ -9,7 +8,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading; -using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Xunit; using Xunit.Abstractions; @@ -658,45 +657,59 @@ public void MultipleListeners_PublishingInstruments() AssertInitialEnumerationCompleteEventPresent(events2); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] + public static bool IsNotBrowserAndRemoteExecuteSupported => PlatformDetection.IsNotBrowser && RemoteExecutor.IsSupported; + + [ConditionalFact(typeof(MetricEventSourceTests), nameof(IsNotBrowserAndRemoteExecuteSupported))] [OuterLoop("Slow and has lots of console spew")] public void EventSourcePublishesTimeSeriesWithEmptyMetadata() { - using Meter meter = new Meter("TestMeter1", null, new TagList() { { "Mk1", "Mv1" }, { "Mk2", "Mv2" } }, new object()); - Counter c = meter.CreateCounter("counter1"); - int counterState = 3; - ObservableCounter oc = meter.CreateObservableCounter("observableCounter1", () => { counterState += 7; return counterState; }); - int gaugeState = 0; - ObservableGauge og = meter.CreateObservableGauge("observableGauge1", () => { gaugeState += 9; return gaugeState; }); - Histogram h = meter.CreateHistogram("histogram1"); - UpDownCounter udc = meter.CreateUpDownCounter("upDownCounter1"); - int upDownCounterState = 0; - ObservableUpDownCounter oudc = meter.CreateObservableUpDownCounter("observableUpDownCounter1", () => { upDownCounterState -= 11; return upDownCounterState; }); - - EventWrittenEventArgs[] events; - using (MetricsEventListener listener = new MetricsEventListener(_output, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter1")) + RemoteExecutor.Invoke(static () => { - listener.WaitForCollectionStop(s_waitForEventTimeout, 1); - c.Add(5); - h.Record(19); - udc.Add(-33); - listener.WaitForCollectionStop(s_waitForEventTimeout, 2); - c.Add(12); - h.Record(26); - udc.Add(-40); - listener.WaitForCollectionStop(s_waitForEventTimeout, 3); - events = listener.Events.ToArray(); - } + CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("fi-FI"); - AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h, udc, oudc); - AssertInitialEnumerationCompleteEventPresent(events); - AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", ("5", "5"), ("12", "17")); - AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", ("", "10"), ("7", "17")); - AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "9", "18"); - AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", ("0.5=19;0.95=19;0.99=19", "1", "19"), ("0.5=26;0.95=26;0.99=26", "1", "26")); - AssertUpDownCounterEventsPresent(events, meter.Name, udc.Name, "", "", ("-33", "-33"), ("-40", "-73")); - AssertUpDownCounterEventsPresent(events, meter.Name, oudc.Name, "", "", ("", "-11"), ("-11", "-22")); - AssertCollectStartStopEventsPresent(events, IntervalSecs, 3); + using Meter meter = new Meter("TestMeter1", null, new TagList() { { "Mk1", "Mv1" }, { "Mk2", "Mv2" } }, new object()); + Counter c = meter.CreateCounter("counter1"); + int counterState = 3; + ObservableCounter oc = meter.CreateObservableCounter("observableCounter1", () => { counterState += 7; return counterState; }); + int gaugeState = 0; + ObservableGauge og = meter.CreateObservableGauge("observableGauge1", () => { gaugeState += 9; return gaugeState; }); + Histogram h = meter.CreateHistogram("histogram1"); + UpDownCounter udc = meter.CreateUpDownCounter("upDownCounter1"); + int upDownCounterState = 0; + ObservableUpDownCounter oudc = meter.CreateObservableUpDownCounter("observableUpDownCounter1", () => { upDownCounterState -= 11; return upDownCounterState; }); + + EventWrittenEventArgs[] events; + using (MetricsEventListener listener = new MetricsEventListener(NullTestOutputHelper.Instance, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter1")) + { + listener.WaitForCollectionStop(s_waitForEventTimeout, 1); + c.Add(5); + h.Record(19); + udc.Add(-33); + listener.WaitForCollectionStop(s_waitForEventTimeout, 2); + c.Add(12); + h.Record(26); + udc.Add(-40); + listener.WaitForCollectionStop(s_waitForEventTimeout, 3); + events = listener.Events.ToArray(); + } + + AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h, udc, oudc); + AssertInitialEnumerationCompleteEventPresent(events); + AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", ("5", "5"), ("12", "17")); + AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", ("", "10"), ("7", "17")); + AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "9", "18"); + AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", ("0.5=19;0.95=19;0.99=19", "1", "19"), ("0.5=26;0.95=26;0.99=26", "1", "26")); + AssertUpDownCounterEventsPresent(events, meter.Name, udc.Name, "", "", ("-33", "-33"), ("-40", "-73")); + AssertUpDownCounterEventsPresent(events, meter.Name, oudc.Name, "", "", ("", "-11"), ("-11", "-22")); + AssertCollectStartStopEventsPresent(events, IntervalSecs, 3); + }).Dispose(); + } + + private sealed class NullTestOutputHelper : ITestOutputHelper + { + public static NullTestOutputHelper Instance { get; } = new(); + public void WriteLine(string message) { } + public void WriteLine(string format, params object[] args) { } } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] @@ -1470,7 +1483,7 @@ private static string FormatTags(IEnumerable>? tag return sb.ToString(); } - private void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private static void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var beginReportEvents = events.Where(e => e.EventName == "BeginInstrumentReporting").Select(e => new @@ -1502,7 +1515,7 @@ private void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] Assert.Equal(expectedInstruments.Length, beginReportEvents.Length); } - private void AssertEndInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private static void AssertEndInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var beginReportEvents = events.Where(e => e.EventName == "EndInstrumentReporting").Select(e => new @@ -1534,27 +1547,27 @@ private void AssertEndInstrumentReportingEventsPresent(EventWrittenEventArgs[] e Assert.Equal(expectedInstruments.Length, beginReportEvents.Length); } - private void AssertInitialEnumerationCompleteEventPresent(EventWrittenEventArgs[] events, int eventsCount = 1) + private static void AssertInitialEnumerationCompleteEventPresent(EventWrittenEventArgs[] events, int eventsCount = 1) { Assert.Equal(eventsCount, events.Where(e => e.EventName == "InitialInstrumentEnumerationComplete").Count()); } - private void AssertTimeSeriesLimitPresent(EventWrittenEventArgs[] events) + private static void AssertTimeSeriesLimitPresent(EventWrittenEventArgs[] events) { Assert.Equal(1, events.Where(e => e.EventName == "TimeSeriesLimitReached").Count()); } - private void AssertTimeSeriesLimitNotPresent(EventWrittenEventArgs[] events) + private static void AssertTimeSeriesLimitNotPresent(EventWrittenEventArgs[] events) { Assert.Equal(0, events.Where(e => e.EventName == "TimeSeriesLimitReached").Count()); } - private void AssertHistogramLimitPresent(EventWrittenEventArgs[] events) + private static void AssertHistogramLimitPresent(EventWrittenEventArgs[] events) { Assert.Equal(1, events.Where(e => e.EventName == "HistogramLimitReached").Count()); } - private void AssertInstrumentPublishingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private static void AssertInstrumentPublishingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var publishEvents = events.Where(e => e.EventName == "InstrumentPublished").Select(e => new @@ -1586,19 +1599,19 @@ private void AssertInstrumentPublishingEventsPresent(EventWrittenEventArgs[] eve Assert.Equal(expectedInstruments.Length, publishEvents.Length); } - private void AssertCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private static void AssertCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { AssertGenericCounterEventsPresent("CounterRateValuePublished", events, meterName, instrumentName, tags, expectedUnit, expected); } - private void AssertUpDownCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private static void AssertUpDownCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { AssertGenericCounterEventsPresent("UpDownCounterRateValuePublished", events, meterName, instrumentName, tags, expectedUnit, expected); } - private void AssertGenericCounterEventsPresent(string eventName, EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private static void AssertGenericCounterEventsPresent(string eventName, EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { var counterEvents = events.Where(e => e.EventName == eventName).Select(e => @@ -1622,7 +1635,7 @@ private void AssertGenericCounterEventsPresent(string eventName, EventWrittenEve } } - private void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) + private static void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) { var counterEvents = events.Where(e => e.EventName == "CounterRateValuePublished").Select(e => new @@ -1636,7 +1649,7 @@ private void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events, strin Assert.Equal(0, filteredEvents.Length); } - private void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private static void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params string[] expectedValues) { var counterEvents = events.Where(e => e.EventName == "GaugeValuePublished").Select(e => @@ -1658,7 +1671,7 @@ private void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, string met } } - private void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private static void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string, string)[] expected) { var counterEvents = events.Where(e => e.EventName == "HistogramValuePublished").Select(e => @@ -1684,7 +1697,7 @@ private void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, string } } - private void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) + private static void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) { var counterEvents = events.Where(e => e.EventName == "HistogramValuePublished").Select(e => new @@ -1697,7 +1710,7 @@ private void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] events, str var filteredEvents = counterEvents.Where(e => e.MeterName == meterName && e.InstrumentName == instrumentName && e.Tags == tags).ToArray(); Assert.Equal(0, filteredEvents.Length); } - private void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] events, double expectedIntervalSecs, int expectedPairs) + private static void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] events, double expectedIntervalSecs, int expectedPairs) { int startEventsSeen = 0; int stopEventsSeen = 0; @@ -1726,7 +1739,7 @@ private void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] events, Assert.Equal(expectedPairs, stopEventsSeen); } - private void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] events) + private static void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] events) { var errorEvents = events.Where(e => e.EventName == "ObservableInstrumentCallbackError").Select(e => new @@ -1737,7 +1750,7 @@ private void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] events Assert.Contains("Example user exception", errorEvents[0].ErrorText); } - private void AssertMultipleSessionsConfiguredIncorrectlyErrorEventsPresent(EventWrittenEventArgs[] events, + private static void AssertMultipleSessionsConfiguredIncorrectlyErrorEventsPresent(EventWrittenEventArgs[] events, string expectedMaxHistograms, string actualMaxHistograms, string expectedMaxTimeSeries, string actualMaxTimeSeries, string expectedRefreshInterval, string actualRefreshInterval) { From db167d9d3fa127cb12e7a4c6a49cb4c1ce7a4279 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 13:35:25 +0100 Subject: [PATCH 014/151] [release/8.0-staging] Add ProducesNetCoreAssets property to Publishing.props (#98988) * add property to publishing.props * rename to ProducesDotNetReleaseShippingAssets * Update Publishing.props * Set assets manifest metadata for assets that get shipped with .NET release (#98824) * add metadata to manifest * set in ItemDefinitionGroup * remove from items * comment --------- Co-authored-by: MilenaHristova Co-authored-by: Viktor Hofer --- eng/Publishing.props | 7 ++++--- src/installer/prepare-artifacts.proj | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/eng/Publishing.props b/eng/Publishing.props index 920e79cbbd2..8b796225f82 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,6 +1,7 @@ - + - 3 + true - \ No newline at end of file + + diff --git a/src/installer/prepare-artifacts.proj b/src/installer/prepare-artifacts.proj index 5413e825c95..a9e0931c116 100644 --- a/src/installer/prepare-artifacts.proj +++ b/src/installer/prepare-artifacts.proj @@ -23,6 +23,8 @@ + + @@ -56,6 +58,16 @@ + + + + DotNetReleaseShipping=true + + + 8.0.0-rtm.23523.2 - 2.2.5-ci.444313 + 2.3.5 8.0.0-alpha.1.23527.1 16.0.5-alpha.1.23566.1 From 18b5fe657919af3f94b21aaa90385f7a2d9c052f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Mar 2024 22:05:05 -0400 Subject: [PATCH 021/151] Fix AsyncVoidMethodBuilder race condition around SynchronizationContext (#99640) This fixes a long-standing issue we've seen sporadically over the years but for which we just got a solid repro; the symptom is a sporadic unhandled null reference exception that crashes an app when using an async void method builder and a non-default SynchronizationContext. The issue is that, because of how state management is handled in the builder, the builder itself can be cleared while its SetResult method is running, and that means two reads of the _synchronizationContext field can end up returning a non-null value followed by a null value. The fix is to just cache the field into a local before completing the builder, and then only use the local state after. Co-authored-by: Stephen Toub --- .../AsyncVoidMethodBuilder.cs | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs index c4719b86132..0caebcee620 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs @@ -80,13 +80,27 @@ public void SetResult() TplEventSource.Log.TraceOperationEnd(this.Task.Id, AsyncCausalityStatus.Completed); } + // Grab the context. Calling SetResult will complete the builder which can cause the state + // to be cleared out of the builder, so we can't touch anything on this builder after calling Set*. + // This clearing is done as part of the AsyncStateMachineBox.MoveNext method after it calls + // MoveNext on the state machine: it's possible to have a chain of events like this: + // Thread 1: Calls AsyncStateMachineBox.MoveNext, which calls StateMachine.MoveNext. + // Thread 1: StateMachine.MoveNext hooks up a continuation and returns + // Thread 2: That continuation runs and calls AsyncStateMachineBox.MoveNext, which calls SetResult on the builder (below) + // which will result in the state machine task being marked completed. + // Thread 1: The original AsyncStateMachineBox.MoveNext call continues and sees that the task is now completed + // Thread 1: Clears the builder + // Thread 2: Continues in this call to AsyncVoidMethodBuilder. If it touches anything on this instance, it will be cleared. + SynchronizationContext? context = _synchronizationContext; + // Mark the builder as completed. As this is a void-returning method, this mostly // doesn't matter, but it can affect things like debug events related to finalization. + // Marking the task completed will also then enable the MoveNext code to clear state. _builder.SetResult(); - if (_synchronizationContext != null) + if (context != null) { - NotifySynchronizationContextOfCompletion(); + NotifySynchronizationContextOfCompletion(context); } } @@ -106,17 +120,18 @@ public void SetException(Exception exception) TplEventSource.Log.TraceOperationEnd(this.Task.Id, AsyncCausalityStatus.Error); } - if (_synchronizationContext != null) + SynchronizationContext? context = _synchronizationContext; + if (context != null) { // If we captured a synchronization context, Post the throwing of the exception to it // and decrement its outstanding operation count. try { - Task.ThrowAsync(exception, targetContext: _synchronizationContext); + Task.ThrowAsync(exception, targetContext: context); } finally { - NotifySynchronizationContextOfCompletion(); + NotifySynchronizationContextOfCompletion(context); } } else @@ -132,12 +147,12 @@ public void SetException(Exception exception) } /// Notifies the current synchronization context that the operation completed. - private void NotifySynchronizationContextOfCompletion() + private static void NotifySynchronizationContextOfCompletion(SynchronizationContext context) { - Debug.Assert(_synchronizationContext != null, "Must only be used with a non-null context."); + Debug.Assert(context != null, "Must only be used with a non-null context."); try { - _synchronizationContext.OperationCompleted(); + context.OperationCompleted(); } catch (Exception exc) { From f09cd44a09a6d311a100404810be049f2caf7a46 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 07:52:28 -0700 Subject: [PATCH 022/151] Always keep global symbols on ApplePlatforms (#99650) Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com> --- .../BuildIntegration/Microsoft.NETCore.Native.targets | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index e379eb457bd..32f6be8d9c8 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -358,7 +358,6 @@ The .NET Foundation licenses this file to you under the MIT license. <_IgnoreLinkerWarnings>false <_IgnoreLinkerWarnings Condition="'$(_IsApplePlatform)' == 'true'">true - <_StripFlag Condition="'$(_IsApplePlatform)' == 'true' and '$(IlcExportUnmanagedEntrypoints)' == 'true'">-x @@ -386,7 +385,7 @@ The .NET Foundation licenses this file to you under the MIT license. + strip -no_code_signature_warning -x "$(NativeBinary)"" /> Date: Wed, 13 Mar 2024 08:46:40 -0700 Subject: [PATCH 023/151] Delete Decrypt_Pkcs1_ErrorsForInvalidPadding This test has a small random chance of failure because of non-determinism. The test Decrypt_Pkcs1_BadPadding covers this scenario and is properly deterministic, so let's go ahead and delete the test that has a chance of failing. Co-authored-by: Kevin Jones --- .../RSA/EncryptDecrypt.cs | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs index 5778b66d043..0aaffebe542 100644 --- a/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs +++ b/src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/EncryptDecrypt.cs @@ -718,26 +718,6 @@ public void NotSupportedValueMethods() } } - [ConditionalTheory] - [InlineData(new byte[] { 1, 2, 3, 4 })] - [InlineData(new byte[0])] - public void Decrypt_Pkcs1_ErrorsForInvalidPadding(byte[] data) - { - if (data.Length == 0 && !PlatformSupportsEmptyRSAEncryption) - { - throw new SkipTestException("Platform does not support RSA encryption of empty data."); - } - - using (RSA rsa = RSAFactory.Create(TestData.RSA2048Params)) - { - byte[] encrypted = Encrypt(rsa, data, RSAEncryptionPadding.Pkcs1); - encrypted[1] ^= 0xFF; - - // PKCS#1, the data, and the key are all deterministic so this should always throw an exception. - Assert.ThrowsAny(() => Decrypt(rsa, encrypted, RSAEncryptionPadding.Pkcs1)); - } - } - [Fact] public void Decrypt_Pkcs1_BadPadding() { From 86f0444770f3950bcbb2ae435958dba80ad08fca Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:45:55 -0700 Subject: [PATCH 024/151] Update dependencies from https://github.com/dotnet/emsdk build (#99716) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24163.1 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 97380116615..50349586bd3 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7bad61f7c23..b590e3f06ed 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - 1639670c6547454278f51afc5c74e20f8acc7abd + 08a90ca2c88b17f1b5d081318354a41db0882cff - + https://github.com/dotnet/emsdk - 1639670c6547454278f51afc5c74e20f8acc7abd + 08a90ca2c88b17f1b5d081318354a41db0882cff From c412efbe9ced452289c0b3bed2fc01ba2d2d1440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Wed, 13 Mar 2024 18:32:22 -0700 Subject: [PATCH 025/151] Rename MSBuild property MicrosoftNativeQuicMsQuicVersion -> MicrosoftNativeQuicMsQuicSchannelVersion (#99714) --- eng/Versions.props | 2 +- src/libraries/System.Net.Quic/src/System.Net.Quic.csproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 57ccfbd0c1f..609fc2931ba 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -219,7 +219,7 @@ 8.0.0-rtm.23523.2 - 2.2.3 + 2.2.3 8.0.0-alpha.1.23527.1 16.0.5-alpha.1.23566.1 diff --git a/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj b/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj index ba05cef37b5..d6c063e216e 100644 --- a/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj +++ b/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj @@ -148,7 +148,7 @@ GeneratePathProperty="true" /> From e3734249509ed793382f7538bd55b5bebfb0f1cc Mon Sep 17 00:00:00 2001 From: Jeremy Barton Date: Thu, 14 Mar 2024 09:57:37 -0700 Subject: [PATCH 026/151] Fixing SignedXml.CheckSignature for enveloped signature with `#xpointer(/)` Reference This additionally improves support for URI-less Reference elements. Co-authored-by: Samo Prelog Co-authored-by: Kevin Jones --- .../System.Security.Cryptography.Xml.csproj | 2 + .../Security/Cryptography/Xml/Reference.cs | 25 ++++++++--- .../tests/SignedXmlTest.cs | 41 +++++++++++++++++++ 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Security.Cryptography.Xml/src/System.Security.Cryptography.Xml.csproj b/src/libraries/System.Security.Cryptography.Xml/src/System.Security.Cryptography.Xml.csproj index 5aea1a156c9..dacbf625586 100644 --- a/src/libraries/System.Security.Cryptography.Xml/src/System.Security.Cryptography.Xml.csproj +++ b/src/libraries/System.Security.Cryptography.Xml/src/System.Security.Cryptography.Xml.csproj @@ -4,6 +4,8 @@ true $(NoWarn);CA1850 true + true + 1 Provides classes to support the creation and validation of XML digital signatures. The classes in this namespace implement the World Wide Web Consortium Recommendation, "XML-Signature Syntax and Processing", described at http://www.w3.org/TR/xmldsig-core/. Commonly Used Types: diff --git a/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs b/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs index ba3a46bc7a7..4e146c45e07 100644 --- a/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs +++ b/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs @@ -266,18 +266,31 @@ public void LoadXml(XmlElement value) // let the transform read the children of the transformElement for data transform.LoadInnerXml(transformElement.ChildNodes); // Hack! this is done to get around the lack of here() function support in XPath - if (transform is XmlDsigEnvelopedSignatureTransform - && _uri != null && (_uri.Length == 0 || _uri[0] == '#')) + if (transform is XmlDsigEnvelopedSignatureTransform) { // Walk back to the Signature tag. Find the nearest signature ancestor // Signature-->SignedInfo-->Reference-->Transforms-->Transform XmlNode? signatureTag = transformElement.SelectSingleNode("ancestor::ds:Signature[1]", nsm); // Resolve the reference to get starting point for position calculation. - XmlNode? referenceTarget = - _uri.Length == 0 - ? transformElement.OwnerDocument - : SignedXml!.GetIdElement(transformElement.OwnerDocument, Utils.GetIdFromLocalUri(_uri, out bool _)); + // This needs to match the way CalculateSignature resolves URI references. + XmlNode? referenceTarget = null; + if (_uri == null || _uri.Length == 0) + { + referenceTarget = transformElement.OwnerDocument; + } + else if (_uri[0] == '#') + { + string idref = Utils.ExtractIdFromLocalUri(_uri); + if (idref == "xpointer(/)") + { + referenceTarget = transformElement.OwnerDocument; + } + else + { + referenceTarget = SignedXml!.GetIdElement(transformElement.OwnerDocument, idref); + } + } XmlNodeList? signatureList = referenceTarget?.SelectNodes(".//ds:Signature", nsm); if (signatureList != null) diff --git a/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs b/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs index 3db8c44aed8..bd22c5b835e 100644 --- a/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs +++ b/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs @@ -9,6 +9,7 @@ // (C) 2002, 2003 Motus Technologies Inc. (http://www.motus.com) // Copyright (C) 2004-2005, 2008 Novell, Inc (http://www.novell.com) +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Net; @@ -1993,5 +1994,45 @@ public void CheckSignatureHandlesIncorrectOrTamperedReferenceWithMultipleEnvelop Assert.False(subject.CheckSignature()); } + + public static object[][] EnvelopedSignatureWithRootXpointerReference = new object[][] + { + new object[] { true, """HiSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=dqcBmS1ZvDJNhmCEgobpAb+A2XaiuB69dfGIhisZvqoxaWqAqv/0w49jp38+usJ5t3wcq3aMC631QE8iln+lHWrarojDMDWLa00isv3oE3q9UgOIV9e6MUSoRTTvQkmlK/LSYV9T/SKx6h03vLLcIkUMXaTkC/n2kthlJTGkLbU=t6qV1iTlkCPoaIeOTvnDczQv5pytUxMoyNXws5vaMQYxfJMKos47dvmiLtfWUDLYXFX3Yf/JMC14plJw2JA5jLrlHLnZj/vCjRtXckmWW/wGYewXUqrgR1CytStUeQKj9mNsi76erukua10UhzIrWG+H6YQ/qS4AMMJZU6jBvO0=AQAB""" }, + new object[] { false, """Tempered worldSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=dqcBmS1ZvDJNhmCEgobpAb+A2XaiuB69dfGIhisZvqoxaWqAqv/0w49jp38+usJ5t3wcq3aMC631QE8iln+lHWrarojDMDWLa00isv3oE3q9UgOIV9e6MUSoRTTvQkmlK/LSYV9T/SKx6h03vLLcIkUMXaTkC/n2kthlJTGkLbU=t6qV1iTlkCPoaIeOTvnDczQv5pytUxMoyNXws5vaMQYxfJMKos47dvmiLtfWUDLYXFX3Yf/JMC14plJw2JA5jLrlHLnZj/vCjRtXckmWW/wGYewXUqrgR1CytStUeQKj9mNsi76erukua10UhzIrWG+H6YQ/qS4AMMJZU6jBvO0=AQAB""" }, + }; + + [Theory] + [MemberData(nameof(EnvelopedSignatureWithRootXpointerReference))] + public void CheckSignatureHandlesEnvelopedSignatureWithRootXpointerReference(bool isValid, string xml) + { + XmlDocument xmlDoc = new (); + xmlDoc.LoadXml(xml); + SignedXml signedXml = new (xmlDoc); + signedXml.LoadXml(xmlDoc.GetElementsByTagName("Signature", SignedXml.XmlDsigNamespaceUrl)[0] as XmlElement); + + Assert.Equal(isValid, signedXml.CheckSignature()); + } + + + public static object[][] EnvelopedSignatureWithEmptyReference = new object[][] + { + new object[] { true, """HiSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=CiB9jgIS7+Wq+lpyzCGsBZQcQ2BxqQuEU9VCvb3Li5jMtjwRV1bMO+4Wfnb4VWhEtEUq6NdiVGXhC1xvtVLnnLDX7CD/jG6NvM1Yd0/rf0UUceBhzYLFE9HLsopsBmmm3t8FO6ZtRr1QqKM0XDaQleGK9vYd2m2Jq8OR3r/w4OY=vcM1wQVmLB9DwdnAym8l8nw63/HlTVzgTDhIwNzWPhsPE/qr2wlK4TEQ3rjU+RAdNytfFNCnuuh75ZVMjAWCV9h6VDlp0DOvBhb6GenhymtTAdJJKzBXKJP6mNPga9cPOP31IZ36Ui00G3fjBBPrHa7nStludgL9Wi0dBU28DjU=AQAB""" }, + new object[] { false, """HISVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=CiB9jgIS7+Wq+lpyzCGsBZQcQ2BxqQuEU9VCvb3Li5jMtjwRV1bMO+4Wfnb4VWhEtEUq6NdiVGXhC1xvtVLnnLDX7CD/jG6NvM1Yd0/rf0UUceBhzYLFE9HLsopsBmmm3t8FO6ZtRr1QqKM0XDaQleGK9vYd2m2Jq8OR3r/w4OY=vcM1wQVmLB9DwdnAym8l8nw63/HlTVzgTDhIwNzWPhsPE/qr2wlK4TEQ3rjU+RAdNytfFNCnuuh75ZVMjAWCV9h6VDlp0DOvBhb6GenhymtTAdJJKzBXKJP6mNPga9cPOP31IZ36Ui00G3fjBBPrHa7nStludgL9Wi0dBU28DjU=AQAB""" }, + }; + + [Theory] + [MemberData(nameof(EnvelopedSignatureWithEmptyReference))] + public void CheckSignatureHandlesEnvelopedSignatureWithEmptyReference(bool isValid, string xml) + { + XmlDocument xmlDoc = new (); + xmlDoc.LoadXml(xml); + SignedXml signedXml = new (xmlDoc); + signedXml.LoadXml(xmlDoc.GetElementsByTagName("Signature", SignedXml.XmlDsigNamespaceUrl)[0] as XmlElement); + + // without this, CheckSignature throws + ((Reference)signedXml.SignedInfo.References[0]).TransformChain[0].LoadInput(xmlDoc); + + Assert.Equal(isValid, signedXml.CheckSignature()); + } } } From 50c3c8697c148b8fa88ad6a7174fbdb2811c6dfe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:40:06 -0700 Subject: [PATCH 027/151] [release/8.0-staging] disable optimizations for PopCount (#99832) * disable optimizations for PopCount avoid using an optimization which might fail on non-SSE4 cpus. * remove whitespace for jit-format --------- Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com> --- src/coreclr/jit/utils.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 2e1c0a52a3d..4c9fe6479ec 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -3192,6 +3192,11 @@ uint32_t BitOperations::Log2(uint64_t value) // Return Value: // The population count (number of bits set) of value // +#if defined(_MSC_VER) +// Disable optimizations for PopCount to avoid the compiler from generating intrinsics +// not supported on all platforms. +#pragma optimize("", off) +#endif // _MSC_VER uint32_t BitOperations::PopCount(uint32_t value) { #if defined(_MSC_VER) @@ -3244,6 +3249,9 @@ uint32_t BitOperations::PopCount(uint64_t value) return static_cast(result); #endif } +#if defined(_MSC_VER) +#pragma optimize("", on) +#endif // _MSC_VER //------------------------------------------------------------------------ // BitOperations::ReverseBits: Reverses the bits in an integer value From ff80834c258fbc9b7cd25a1703d16fe1fb315205 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:25:42 -0700 Subject: [PATCH 028/151] [release/8.0-staging] Handle NativeOverlapped* coming from both the Windows or Portable thread pool in NativeRuntimeEventSource (#99656) --- ...untimeEventSource.Threading.NativeSinks.cs | 27 ++++++++++++++----- .../NativeRuntimeEventSource.Threading.cs | 27 ++++++++++++++----- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs index ad7ee54ae22..1dab92fc363 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs @@ -194,7 +194,7 @@ public unsafe void ThreadPoolWorkerThreadAdjustmentStats( [Event(63, Level = EventLevel.Verbose, Message = Messages.IOEnqueue, Task = Tasks.ThreadPool, Opcode = Opcodes.IOEnqueue, Version = 0, Keywords = Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)] private unsafe void ThreadPoolIOEnqueue( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary bool MultiDequeues, ushort ClrInstanceID = DefaultClrInstanceId) { @@ -207,9 +207,14 @@ public unsafe void ThreadPoolIOEnqueue(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIOEnqueue( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(), + overlapped, false); } } @@ -231,7 +236,7 @@ public void ThreadPoolIOEnqueue(RegisteredWaitHandle registeredWaitHandle) [Event(64, Level = EventLevel.Verbose, Message = Messages.IO, Task = Tasks.ThreadPool, Opcode = Opcodes.IODequeue, Version = 0, Keywords = Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)] private unsafe void ThreadPoolIODequeue( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary ushort ClrInstanceID = DefaultClrInstanceId) { LogThreadPoolIODequeue(NativeOverlapped, Overlapped, ClrInstanceID); @@ -243,9 +248,14 @@ public unsafe void ThreadPoolIODequeue(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIODequeue( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode()); + overlapped); } } @@ -277,16 +287,21 @@ public unsafe void ThreadPoolIOPack(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIOPack( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode()); + overlapped); } } [Event(65, Level = EventLevel.Verbose, Message = Messages.IO, Task = Tasks.ThreadPool, Opcode = Opcodes.IOPack, Version = 0, Keywords = Keywords.ThreadingKeyword)] private unsafe void ThreadPoolIOPack( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary ushort ClrInstanceID = DefaultClrInstanceId) { LogThreadPoolIOPack(NativeOverlapped, Overlapped, ClrInstanceID); diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs index c7a6d96aab2..b883d9a6aa7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs @@ -316,7 +316,7 @@ public unsafe void ThreadPoolWorkerThreadAdjustmentStats( [Event(63, Level = EventLevel.Verbose, Message = Messages.IOEnqueue, Task = Tasks.ThreadPool, Opcode = Opcodes.IOEnqueue, Version = 0, Keywords = Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)] private unsafe void ThreadPoolIOEnqueue( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary bool MultiDequeues, ushort ClrInstanceID = DefaultClrInstanceId) { @@ -343,9 +343,14 @@ public unsafe void ThreadPoolIOEnqueue(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIOEnqueue( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(), + overlapped, false); } } @@ -368,7 +373,7 @@ public void ThreadPoolIOEnqueue(RegisteredWaitHandle registeredWaitHandle) [Event(64, Level = EventLevel.Verbose, Message = Messages.IO, Task = Tasks.ThreadPool, Opcode = Opcodes.IODequeue, Version = 0, Keywords = Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)] private unsafe void ThreadPoolIODequeue( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary ushort ClrInstanceID = DefaultClrInstanceId) { EventData* data = stackalloc EventData[3]; @@ -390,9 +395,14 @@ public unsafe void ThreadPoolIODequeue(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword | Keywords.ThreadTransferKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIODequeue( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode()); + overlapped); } } @@ -432,9 +442,14 @@ public unsafe void ThreadPoolIOPack(NativeOverlapped* nativeOverlapped) { if (IsEnabled(EventLevel.Verbose, Keywords.ThreadingKeyword)) { +#if TARGET_WINDOWS + IntPtr overlapped = ThreadPool.UseWindowsThreadPool ? 0 : (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#else + IntPtr overlapped = (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode(); +#endif ThreadPoolIOPack( (IntPtr)nativeOverlapped, - (IntPtr)Overlapped.GetOverlappedFromNative(nativeOverlapped).GetHashCode()); + overlapped); } } @@ -442,7 +457,7 @@ public unsafe void ThreadPoolIOPack(NativeOverlapped* nativeOverlapped) [Event(65, Level = EventLevel.Verbose, Message = Messages.IO, Task = Tasks.ThreadPool, Opcode = Opcodes.IOPack, Version = 0, Keywords = Keywords.ThreadingKeyword)] private unsafe void ThreadPoolIOPack( IntPtr NativeOverlapped, - IntPtr Overlapped, + IntPtr Overlapped, // 0 if the Windows thread pool is used, the relevant info could be obtained from the NativeOverlapped* if necessary ushort ClrInstanceID = DefaultClrInstanceId) { EventData* data = stackalloc EventData[3]; From fd8f5b5af7d73f697fe9654ad8ed7953cd597287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:51:07 -0700 Subject: [PATCH 029/151] [release/8.0] disable optimizations for PopCount (#99832) (#99926) * disable optimizations for PopCount avoid using an optimization which might fail on non-SSE4 cpus. * remove whitespace for jit-format --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com> --- src/coreclr/jit/utils.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 2e1c0a52a3d..4c9fe6479ec 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -3192,6 +3192,11 @@ uint32_t BitOperations::Log2(uint64_t value) // Return Value: // The population count (number of bits set) of value // +#if defined(_MSC_VER) +// Disable optimizations for PopCount to avoid the compiler from generating intrinsics +// not supported on all platforms. +#pragma optimize("", off) +#endif // _MSC_VER uint32_t BitOperations::PopCount(uint32_t value) { #if defined(_MSC_VER) @@ -3244,6 +3249,9 @@ uint32_t BitOperations::PopCount(uint64_t value) return static_cast(result); #endif } +#if defined(_MSC_VER) +#pragma optimize("", on) +#endif // _MSC_VER //------------------------------------------------------------------------ // BitOperations::ReverseBits: Reverses the bits in an integer value From cf1182c88b8dc2cd4a63c7697648d3f96031fe65 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 17:17:15 -0700 Subject: [PATCH 030/151] Fix exporting certificate keys on macOS 14.4. Apple changed the error code we get back from a failed data-key export. This caused us to not attempt to export the key using the legacy APIs and assume the key export failed. This pull request adds the additional error code returned from macOS 14.4. Co-authored-by: Kevin Jones --- .../Interop.SecKeyRef.cs | 6 +- .../tests/X509Certificates/CertTests.cs | 102 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs index e7c08596a2c..5186018760a 100644 --- a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs +++ b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs @@ -127,6 +127,10 @@ internal static bool TrySecKeyCopyExternalRepresentation( { const int errSecPassphraseRequired = -25260; + // macOS Sonoma 14.4 started returning errSecInvalidKeyAttributeMask when a key could not be exported + // because it must be exported with a password. + const int errSecInvalidKeyAttributeMask = -67738; + int result = AppleCryptoNative_SecKeyCopyExternalRepresentation( key, out SafeCFDataHandle data, @@ -141,7 +145,7 @@ internal static bool TrySecKeyCopyExternalRepresentation( externalRepresentation = CoreFoundation.CFGetData(data); return true; case kErrorSeeError: - if (Interop.CoreFoundation.GetErrorCode(errorHandle) == errSecPassphraseRequired) + if (Interop.CoreFoundation.GetErrorCode(errorHandle) is errSecPassphraseRequired or errSecInvalidKeyAttributeMask) { externalRepresentation = Array.Empty(); return false; diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs index f0b852c25b1..c6eadbdcc4c 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs @@ -27,6 +27,108 @@ public CertTests(ITestOutputHelper output) _log = output; } + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDsa() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (ECDsa certKey = cert.GetECDsaPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_RSA() + { + using (RSA ca = RSA.Create(2048)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (RSA certKey = cert.GetRSAPrivateKey()) + { + RSAParameters certParameters = certKey.ExportParameters(true); + RSAParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.P, certParameters.P); + AssertExtensions.SequenceEqual(originalParameters.Q, certParameters.Q); + } + } + } + + [Fact] + [SkipOnPlatform(PlatformSupport.MobileAppleCrypto, "DSA is not available")] + public static void PrivateKey_FromCertificate_CanExportPrivate_DSA() + { + DSAParameters originalParameters = DSATestData.GetDSA1024Params(); + + using (DSA ca = DSA.Create()) + { + ca.ImportParameters(originalParameters); + DSAX509SignatureGenerator gen = new DSAX509SignatureGenerator(ca); + X500DistinguishedName dn = new X500DistinguishedName("CN=potatos"); + + CertificateRequest req = new CertificateRequest( + dn, + gen.PublicKey, + HashAlgorithmName.SHA1); + + using (X509Certificate2 cert = req.Create(dn, gen, DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3), [1, 2, 3])) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ca)) + using (DSA certKey = certWithKey.GetDSAPrivateKey()) + { + DSAParameters certParameters = certKey.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.X, certParameters.X); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDiffieHellman() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + using (ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest issuerRequest = new CertificateRequest( + new X500DistinguishedName("CN=root"), + ca, + HashAlgorithmName.SHA256); + + issuerRequest.CertificateExtensions.Add( + new X509BasicConstraintsExtension(true, false, 0, true)); + + CertificateRequest request = new CertificateRequest( + new X500DistinguishedName("CN=potato"), + new PublicKey(ecdh), + HashAlgorithmName.SHA256); + + request.CertificateExtensions.Add( + new X509BasicConstraintsExtension(false, false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.KeyAgreement, true)); + + DateTimeOffset notBefore = DateTimeOffset.UtcNow; + DateTimeOffset notAfter = notBefore.AddDays(30); + byte[] serial = [1, 2, 3, 4, 5, 6, 7, 8]; + + using (X509Certificate2 issuer = issuerRequest.CreateSelfSigned(notBefore, notAfter)) + using (X509Certificate2 cert = request.Create(issuer, notBefore, notAfter, serial)) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ecdh)) + using (ECDiffieHellman certKey = certWithKey.GetECDiffieHellmanPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ecdh.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + [Fact] public static void PublicPrivateKey_IndependentLifetimes_ECDsa() { From f381395b39f7be81f7cb1bc85b80398226aa4b72 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 19 Mar 2024 12:02:53 -0700 Subject: [PATCH 031/151] Fix exporting certificate keys on macOS 14.4. (#99976) Apple changed the error code we get back from a failed data-key export. This caused us to not attempt to export the key using the legacy APIs and assume the key export failed. This pull request adds the additional error code returned from macOS 14.4. Co-authored-by: Kevin Jones --- .../Interop.SecKeyRef.cs | 6 +- .../tests/X509Certificates/CertTests.cs | 102 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs index e7c08596a2c..5186018760a 100644 --- a/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs +++ b/src/libraries/Common/src/Interop/OSX/System.Security.Cryptography.Native.Apple/Interop.SecKeyRef.cs @@ -127,6 +127,10 @@ internal static bool TrySecKeyCopyExternalRepresentation( { const int errSecPassphraseRequired = -25260; + // macOS Sonoma 14.4 started returning errSecInvalidKeyAttributeMask when a key could not be exported + // because it must be exported with a password. + const int errSecInvalidKeyAttributeMask = -67738; + int result = AppleCryptoNative_SecKeyCopyExternalRepresentation( key, out SafeCFDataHandle data, @@ -141,7 +145,7 @@ internal static bool TrySecKeyCopyExternalRepresentation( externalRepresentation = CoreFoundation.CFGetData(data); return true; case kErrorSeeError: - if (Interop.CoreFoundation.GetErrorCode(errorHandle) == errSecPassphraseRequired) + if (Interop.CoreFoundation.GetErrorCode(errorHandle) is errSecPassphraseRequired or errSecInvalidKeyAttributeMask) { externalRepresentation = Array.Empty(); return false; diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs index f0b852c25b1..c6eadbdcc4c 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs @@ -27,6 +27,108 @@ public CertTests(ITestOutputHelper output) _log = output; } + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDsa() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (ECDsa certKey = cert.GetECDsaPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_RSA() + { + using (RSA ca = RSA.Create(2048)) + { + CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + + using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) + using (RSA certKey = cert.GetRSAPrivateKey()) + { + RSAParameters certParameters = certKey.ExportParameters(true); + RSAParameters originalParameters = ca.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.P, certParameters.P); + AssertExtensions.SequenceEqual(originalParameters.Q, certParameters.Q); + } + } + } + + [Fact] + [SkipOnPlatform(PlatformSupport.MobileAppleCrypto, "DSA is not available")] + public static void PrivateKey_FromCertificate_CanExportPrivate_DSA() + { + DSAParameters originalParameters = DSATestData.GetDSA1024Params(); + + using (DSA ca = DSA.Create()) + { + ca.ImportParameters(originalParameters); + DSAX509SignatureGenerator gen = new DSAX509SignatureGenerator(ca); + X500DistinguishedName dn = new X500DistinguishedName("CN=potatos"); + + CertificateRequest req = new CertificateRequest( + dn, + gen.PublicKey, + HashAlgorithmName.SHA1); + + using (X509Certificate2 cert = req.Create(dn, gen, DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3), [1, 2, 3])) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ca)) + using (DSA certKey = certWithKey.GetDSAPrivateKey()) + { + DSAParameters certParameters = certKey.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.X, certParameters.X); + } + } + } + + [Fact] + public static void PrivateKey_FromCertificate_CanExportPrivate_ECDiffieHellman() + { + using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) + using (ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256)) + { + CertificateRequest issuerRequest = new CertificateRequest( + new X500DistinguishedName("CN=root"), + ca, + HashAlgorithmName.SHA256); + + issuerRequest.CertificateExtensions.Add( + new X509BasicConstraintsExtension(true, false, 0, true)); + + CertificateRequest request = new CertificateRequest( + new X500DistinguishedName("CN=potato"), + new PublicKey(ecdh), + HashAlgorithmName.SHA256); + + request.CertificateExtensions.Add( + new X509BasicConstraintsExtension(false, false, 0, true)); + request.CertificateExtensions.Add( + new X509KeyUsageExtension(X509KeyUsageFlags.KeyAgreement, true)); + + DateTimeOffset notBefore = DateTimeOffset.UtcNow; + DateTimeOffset notAfter = notBefore.AddDays(30); + byte[] serial = [1, 2, 3, 4, 5, 6, 7, 8]; + + using (X509Certificate2 issuer = issuerRequest.CreateSelfSigned(notBefore, notAfter)) + using (X509Certificate2 cert = request.Create(issuer, notBefore, notAfter, serial)) + using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ecdh)) + using (ECDiffieHellman certKey = certWithKey.GetECDiffieHellmanPrivateKey()) + { + ECParameters certParameters = certKey.ExportParameters(true); + ECParameters originalParameters = ecdh.ExportParameters(true); + AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); + } + } + } + [Fact] public static void PublicPrivateKey_IndependentLifetimes_ECDsa() { From c27b2033c5bc66749a5d290841a0fbee5a5791d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Tue, 19 Mar 2024 17:45:15 -0700 Subject: [PATCH 032/151] [release/8.0] Upgrade zlib to 1.3.1 (#99473) * [8.0] Upgrade zlib to 1.3.1 * [PATCH] Make zlib compile clean against C4244 clang equivalent is "implicit-int-conversion" warning The change to deflate.c is legal because 'len' has an upper bound of MAX_STORED, which means it fits cleanly into a 16-bit integer. So writing out 2x 8-bit values will not result in data loss. The change to trees.c is legal because within this loop, 'count' is intended to have an upper bound of 138, with the target assignment only executing if 'count' is bounded by 4. Neither the 'count' local in isolation nor the addition that's part of the target line is expected to result in integer overflow. But even if it did, that's a matter for a different warning code and doesn't impact the correctness of the narrowing cast being considered here. Author: Levi Broderick * Update cgmanifest.json and THIRD-PARTY-NOTICES.TXT * Bring back patches comment, remove unnecessary file removal comment. --- THIRD-PARTY-NOTICES.TXT | 2 +- src/native/external/cgmanifest.json | 2 +- src/native/external/zlib-version.txt | 14 +- src/native/external/zlib/CMakeLists.txt | 45 +- src/native/external/zlib/ChangeLog | 30 +- src/native/external/zlib/FAQ | 5 +- src/native/external/zlib/Makefile.in | 22 +- src/native/external/zlib/README | 19 +- src/native/external/zlib/adler32.c | 32 +- src/native/external/zlib/compress.c | 21 +- src/native/external/zlib/configure | 31 +- src/native/external/zlib/crc32.c | 248 ++++----- src/native/external/zlib/deflate.c | 609 ++++++++++------------- src/native/external/zlib/deflate.h | 51 +- src/native/external/zlib/gzclose.c | 4 +- src/native/external/zlib/gzguts.h | 31 +- src/native/external/zlib/gzlib.c | 113 ++--- src/native/external/zlib/gzread.c | 88 +--- src/native/external/zlib/gzwrite.c | 84 +--- src/native/external/zlib/infback.c | 30 +- src/native/external/zlib/inffast.c | 5 +- src/native/external/zlib/inffast.h | 2 +- src/native/external/zlib/inflate.c | 131 ++--- src/native/external/zlib/inftrees.c | 17 +- src/native/external/zlib/inftrees.h | 10 +- src/native/external/zlib/treebuild.xml | 4 +- src/native/external/zlib/trees.c | 542 +++++++++----------- src/native/external/zlib/uncompr.c | 16 +- src/native/external/zlib/zconf.h | 18 +- src/native/external/zlib/zconf.h.cmakein | 18 +- src/native/external/zlib/zconf.h.in | 18 +- src/native/external/zlib/zlib.3 | 6 +- src/native/external/zlib/zlib.h | 391 +++++++-------- src/native/external/zlib/zlib2ansi | 152 ------ src/native/external/zlib/zutil.c | 60 +-- src/native/external/zlib/zutil.h | 47 +- 36 files changed, 1132 insertions(+), 1786 deletions(-) delete mode 100644 src/native/external/zlib/zlib2ansi diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 4b40333f72c..9b4e777ca47 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -73,7 +73,7 @@ https://github.com/madler/zlib https://zlib.net/zlib_license.html /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.13, October 13th, 2022 + version 1.3.1, January 22nd, 2024 Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler diff --git a/src/native/external/cgmanifest.json b/src/native/external/cgmanifest.json index 4f6264b3c7a..e45d9d90029 100644 --- a/src/native/external/cgmanifest.json +++ b/src/native/external/cgmanifest.json @@ -46,7 +46,7 @@ "Type": "git", "Git": { "RepositoryUrl": "https://github.com/madler/zlib", - "CommitHash": "04f42ceca40f73e2978b50e93806c2a18c1281fc" + "CommitHash": "51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf" } }, "DevelopmentDependency": false diff --git a/src/native/external/zlib-version.txt b/src/native/external/zlib-version.txt index fcac66cc464..5da9869df33 100644 --- a/src/native/external/zlib-version.txt +++ b/src/native/external/zlib-version.txt @@ -1,17 +1,9 @@ -v1.2.13 -(04f42ceca40f73e2978b50e93806c2a18c1281fc) +v1.3.1 +(51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf) -https://github.com/madler/zlib/releases/tag/v1.2.13 +https://github.com/madler/zlib/releases/tag/v1.3.1 We have removed zlib.3.pdf from our local copy, as it is a binary file which is not needed for our compilation. -We have also cherry-picked into our local copy: - -- https://github.com/madler/zlib/commit/e554695638228b846d49657f31eeff0ca4680e8a - - This patch only affects memLevel 9 compression. .NET doesn't currently use this - memLevel, but we'll take this patch out of an abundance of caution just in case - we enable this functionality in a future release. - We have also applied the custom patches under the patches/zlib folder. diff --git a/src/native/external/zlib/CMakeLists.txt b/src/native/external/zlib/CMakeLists.txt index b412dc7feb7..15ceebe787e 100644 --- a/src/native/external/zlib/CMakeLists.txt +++ b/src/native/external/zlib/CMakeLists.txt @@ -1,9 +1,11 @@ -cmake_minimum_required(VERSION 2.4.4) +cmake_minimum_required(VERSION 2.4.4...3.15.0) set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) project(zlib C) -set(VERSION "1.2.13") +set(VERSION "1.3.1") + +option(ZLIB_BUILD_EXAMPLES "Enable Zlib Examples" ON) set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for executables") set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Installation directory for libraries") @@ -148,7 +150,9 @@ if(MINGW) endif(MINGW) add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) +target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) set_target_properties(zlib PROPERTIES SOVERSION 1) @@ -166,7 +170,7 @@ endif() if(UNIX) # On unix-like platforms the library is almost always called libz set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) - if(NOT APPLE) + if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX)) set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") endif() elseif(BUILD_SHARED_LIBS AND WIN32) @@ -193,21 +197,22 @@ endif() #============================================================================ # Example binaries #============================================================================ - -add_executable(example test/example.c) -target_link_libraries(example zlib) -add_test(example example) - -add_executable(minigzip test/minigzip.c) -target_link_libraries(minigzip zlib) - -if(HAVE_OFF64_T) - add_executable(example64 test/example.c) - target_link_libraries(example64 zlib) - set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - add_test(example64 example64) - - add_executable(minigzip64 test/minigzip.c) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") +if(ZLIB_BUILD_EXAMPLES) + add_executable(example test/example.c) + target_link_libraries(example zlib) + add_test(example example) + + add_executable(minigzip test/minigzip.c) + target_link_libraries(minigzip zlib) + + if(HAVE_OFF64_T) + add_executable(example64 test/example.c) + target_link_libraries(example64 zlib) + set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") + add_test(example64 example64) + + add_executable(minigzip64 test/minigzip.c) + target_link_libraries(minigzip64 zlib) + set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") + endif() endif() diff --git a/src/native/external/zlib/ChangeLog b/src/native/external/zlib/ChangeLog index 457526bc6a5..b801a1031ec 100644 --- a/src/native/external/zlib/ChangeLog +++ b/src/native/external/zlib/ChangeLog @@ -1,6 +1,34 @@ ChangeLog file for zlib +Changes in 1.3.1 (22 Jan 2024) +- Reject overflows of zip header fields in minizip +- Fix bug in inflateSync() for data held in bit buffer +- Add LIT_MEM define to use more memory for a small deflate speedup +- Fix decision on the emission of Zip64 end records in minizip +- Add bounds checking to ERR_MSG() macro, used by zError() +- Neutralize zip file traversal attacks in miniunz +- Fix a bug in ZLIB_DEBUG compiles in check_match() +- Various portability and appearance improvements + +Changes in 1.3 (18 Aug 2023) +- Remove K&R function definitions and zlib2ansi +- Fix bug in deflateBound() for level 0 and memLevel 9 +- Fix bug when gzungetc() is used immediately after gzopen() +- Fix bug when using gzflush() with a very small buffer +- Fix crash when gzsetparams() attempted for transparent write +- Fix test/example.c to work with FORCE_STORED +- Rewrite of zran in examples (see zran.c version history) +- Fix minizip to allow it to open an empty zip file +- Fix reading disk number start on zip64 files in minizip +- Fix logic error in minizip argument processing +- Add minizip testing to Makefile +- Read multiple bytes instead of byte-by-byte in minizip unzip.c +- Add memory sanitizer to configure (--memory) +- Various portability improvements +- Various documentation improvements +- Various spelling and typo corrections + Changes in 1.2.13 (13 Oct 2022) - Fix configure issue that discarded provided CC definition - Correct incorrect inputs provided to the CRC functions @@ -1445,7 +1473,7 @@ Changes in 0.99 (27 Jan 96) - fix typo in Make_vms.com (f$trnlnm -> f$getsyi) - in fcalloc, normalize pointer if size > 65520 bytes - don't use special fcalloc for 32 bit Borland C++ -- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc... +- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc. - use Z_BINARY instead of BINARY - document that gzclose after gzdopen will close the file - allow "a" as mode in gzopen diff --git a/src/native/external/zlib/FAQ b/src/native/external/zlib/FAQ index 99b7cf92e45..92f5d3e29fa 100644 --- a/src/native/external/zlib/FAQ +++ b/src/native/external/zlib/FAQ @@ -4,7 +4,7 @@ If your question is not there, please check the zlib home page http://zlib.net/ which may have more recent information. -The lastest zlib FAQ is at http://zlib.net/zlib_faq.html +The latest zlib FAQ is at http://zlib.net/zlib_faq.html 1. Is zlib Y2K-compliant? @@ -14,8 +14,7 @@ The lastest zlib FAQ is at http://zlib.net/zlib_faq.html 2. Where can I get a Windows DLL version? The zlib sources can be compiled without change to produce a DLL. See the - file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the - precompiled DLL are found in the zlib web site at http://zlib.net/ . + file win32/DLL_FAQ.txt in the zlib distribution. 3. Where can I get a Visual Basic interface to zlib? diff --git a/src/native/external/zlib/Makefile.in b/src/native/external/zlib/Makefile.in index 7d2713f4c57..cb8b00a9b07 100644 --- a/src/native/external/zlib/Makefile.in +++ b/src/native/external/zlib/Makefile.in @@ -1,5 +1,5 @@ # Makefile for zlib -# Copyright (C) 1995-2017 Jean-loup Gailly, Mark Adler +# Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler # For conditions of distribution and use, see copyright notice in zlib.h # To compile and test, type: @@ -22,13 +22,13 @@ CFLAGS=-O SFLAGS=-O LDFLAGS= -TEST_LDFLAGS=$(LDFLAGS) -L. libz.a +TEST_LIBS=-L. libz.a LDSHARED=$(CC) CPP=$(CC) -E STATICLIB=libz.a SHAREDLIB=libz.so -SHAREDLIBV=libz.so.1.2.13 +SHAREDLIBV=libz.so.1.3.1 SHAREDLIBM=libz.so.1 LIBS=$(STATICLIB) $(SHAREDLIBV) @@ -282,10 +282,10 @@ placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a -@rmdir objs example$(EXE): example.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ example.o $(TEST_LDFLAGS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example.o $(TEST_LIBS) minigzip$(EXE): minigzip.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ minigzip.o $(TEST_LIBS) examplesh$(EXE): example.o $(SHAREDLIBV) $(CC) $(CFLAGS) -o $@ example.o $(LDFLAGS) -L. $(SHAREDLIBV) @@ -294,10 +294,10 @@ minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) -L. $(SHAREDLIBV) example64$(EXE): example64.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example64.o $(TEST_LIBS) minigzip64$(EXE): minigzip64.o $(STATICLIB) - $(CC) $(CFLAGS) -o $@ minigzip64.o $(TEST_LDFLAGS) + $(CC) $(CFLAGS) $(LDFLAGS) -o $@ minigzip64.o $(TEST_LIBS) install-libs: $(LIBS) -@if [ ! -d $(DESTDIR)$(exec_prefix) ]; then mkdir -p $(DESTDIR)$(exec_prefix); fi @@ -359,8 +359,14 @@ zconf.h.cmakein: $(SRCDIR)zconf.h.in zconf: $(SRCDIR)zconf.h.in cp -p $(SRCDIR)zconf.h.in zconf.h +minizip-test: static + cd contrib/minizip && { CC="$(CC)" CFLAGS="$(CFLAGS)" $(MAKE) test ; cd ../.. ; } + +minizip-clean: + cd contrib/minizip && { $(MAKE) clean ; cd ../.. ; } + mostlyclean: clean -clean: +clean: minizip-clean rm -f *.o *.lo *~ \ example$(EXE) minigzip$(EXE) examplesh$(EXE) minigzipsh$(EXE) \ example64$(EXE) minigzip64$(EXE) \ diff --git a/src/native/external/zlib/README b/src/native/external/zlib/README index ba34d1894a9..c5f917540b6 100644 --- a/src/native/external/zlib/README +++ b/src/native/external/zlib/README @@ -1,6 +1,6 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.2.13 is a general purpose data compression library. All the code is +zlib 1.3.1 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and @@ -29,18 +29,17 @@ PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help. Mark Nelson wrote an article about zlib for the Jan. 1997 issue of Dr. Dobb's Journal; a copy of the article is available at -http://marknelson.us/1997/01/01/zlib-engine/ . +https://marknelson.us/posts/1997/01/01/zlib-engine.html . -The changes made in version 1.2.13 are documented in the file ChangeLog. +The changes made in version 1.3.1 are documented in the file ChangeLog. Unsupported third party contributions are provided in directory contrib/ . -zlib is available in Java using the java.util.zip package, documented at -http://java.sun.com/developer/technicalArticles/Programming/compression/ . +zlib is available in Java using the java.util.zip package. Follow the API +Documentation link at: https://docs.oracle.com/search/?q=java.util.zip . -A Perl interface to zlib written by Paul Marquess is available -at CPAN (Comprehensive Perl Archive Network) sites, including -http://search.cpan.org/~pmqs/IO-Compress-Zlib/ . +A Perl interface to zlib and bzip2 written by Paul Marquess +can be found at https://github.com/pmqs/IO-Compress . A Python interface to zlib written by A.M. Kuchling is available in Python 1.5 and later versions, see @@ -64,7 +63,7 @@ Notes for some targets: - zlib doesn't work with gcc 2.6.3 on a DEC 3000/300LX under OSF/1 2.1 it works when compiled with cc. -- On Digital Unix 4.0D (formely OSF/1) on AlphaServer, the cc option -std1 is +- On Digital Unix 4.0D (formerly OSF/1) on AlphaServer, the cc option -std1 is necessary to get gzprintf working correctly. This is done by configure. - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with @@ -84,7 +83,7 @@ Acknowledgments: Copyright notice: - (C) 1995-2022 Jean-loup Gailly and Mark Adler + (C) 1995-2024 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/src/native/external/zlib/adler32.c b/src/native/external/zlib/adler32.c index d0be4380a39..04b81d29bad 100644 --- a/src/native/external/zlib/adler32.c +++ b/src/native/external/zlib/adler32.c @@ -7,8 +7,6 @@ #include "zutil.h" -local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); - #define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -60,11 +58,7 @@ local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); #endif /* ========================================================================= */ -uLong ZEXPORT adler32_z(adler, buf, len) - uLong adler; - const Bytef *buf; - z_size_t len; -{ +uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { unsigned long sum2; unsigned n; @@ -131,20 +125,12 @@ uLong ZEXPORT adler32_z(adler, buf, len) } /* ========================================================================= */ -uLong ZEXPORT adler32(adler, buf, len) - uLong adler; - const Bytef *buf; - uInt len; -{ +uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len) { return adler32_z(adler, buf, len); } /* ========================================================================= */ -local uLong adler32_combine_(adler1, adler2, len2) - uLong adler1; - uLong adler2; - z_off64_t len2; -{ +local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2) { unsigned long sum1; unsigned long sum2; unsigned rem; @@ -169,18 +155,10 @@ local uLong adler32_combine_(adler1, adler2, len2) } /* ========================================================================= */ -uLong ZEXPORT adler32_combine(adler1, adler2, len2) - uLong adler1; - uLong adler2; - z_off_t len2; -{ +uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, z_off_t len2) { return adler32_combine_(adler1, adler2, len2); } -uLong ZEXPORT adler32_combine64(adler1, adler2, len2) - uLong adler1; - uLong adler2; - z_off64_t len2; -{ +uLong ZEXPORT adler32_combine64(uLong adler1, uLong adler2, z_off64_t len2) { return adler32_combine_(adler1, adler2, len2); } diff --git a/src/native/external/zlib/compress.c b/src/native/external/zlib/compress.c index 2ad5326c14e..f43bacf7ab9 100644 --- a/src/native/external/zlib/compress.c +++ b/src/native/external/zlib/compress.c @@ -19,13 +19,8 @@ memory, Z_BUF_ERROR if there was not enough room in the output buffer, Z_STREAM_ERROR if the level parameter is invalid. */ -int ZEXPORT compress2(dest, destLen, source, sourceLen, level) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; - int level; -{ +int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong sourceLen, int level) { z_stream stream; int err; const uInt max = (uInt)-1; @@ -65,12 +60,8 @@ int ZEXPORT compress2(dest, destLen, source, sourceLen, level) /* =========================================================================== */ -int ZEXPORT compress(dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; -{ +int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong sourceLen) { return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); } @@ -78,9 +69,7 @@ int ZEXPORT compress(dest, destLen, source, sourceLen) If the default memLevel or windowBits for deflateInit() is changed, then this function needs to be updated. */ -uLong ZEXPORT compressBound(sourceLen) - uLong sourceLen; -{ +uLong ZEXPORT compressBound(uLong sourceLen) { return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + (sourceLen >> 25) + 13; } diff --git a/src/native/external/zlib/configure b/src/native/external/zlib/configure index fa4d5daaba9..c55098afc4a 100644 --- a/src/native/external/zlib/configure +++ b/src/native/external/zlib/configure @@ -25,7 +25,7 @@ if test $SRCDIR = "."; then ZINCOUT="-I." SRCDIR="" else - ZINC='-include zconf.h' + ZINC='-I. -include zconf.h' ZINCOUT='-I. -I$(SRCDIR)' SRCDIR="$SRCDIR/" fi @@ -44,9 +44,8 @@ STATICLIB=libz.a # extract zlib version numbers from zlib.h VER=`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < ${SRCDIR}zlib.h` -VER3=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\\.[0-9]*\).*/\1/p' < ${SRCDIR}zlib.h` -VER2=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` -VER1=`sed -n -e '/VERSION "/s/.*"\([0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` +VER3=`echo ${VER}|sed -n -e 's/\([0-9]\{1,\}\(\\.[0-9]\{1,\}\)\{1,2\}\).*/\1/p'` +VER1=`echo ${VER}|sed -n -e 's/\([0-9]\{1,\}\)\\..*/\1/p'` # establish commands for library building if "${CROSS_PREFIX}ar" --version >/dev/null 2>/dev/null || test $? -lt 126; then @@ -90,7 +89,8 @@ build64=0 gcc=0 warn=0 debug=0 -sanitize=0 +address=0 +memory=0 old_cc="$CC" old_cflags="$CFLAGS" OBJC='$(OBJZ) $(OBJG)' @@ -102,7 +102,7 @@ leave() if test "$*" != "0"; then echo "** $0 aborting." | tee -a configure.log fi - rm -f $test.[co] $test $test$shared_ext $test.gcno ./--version + rm -rf $test.[co] $test $test$shared_ext $test.gcno $test.dSYM ./--version echo -------------------- >> configure.log echo >> configure.log echo >> configure.log @@ -141,7 +141,9 @@ case "$1" in -c* | --const) zconst=1; shift ;; -w* | --warn) warn=1; shift ;; -d* | --debug) debug=1; shift ;; - --sanitize) sanitize=1; shift ;; + --sanitize) address=1; shift ;; + --address) address=1; shift ;; + --memory) memory=1; shift ;; *) echo "unknown option: $1" | tee -a configure.log echo "$0 --help for help" | tee -a configure.log @@ -211,8 +213,11 @@ if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then CFLAGS="${CFLAGS} -Wall -Wextra" fi fi - if test $sanitize -eq 1; then - CFLAGS="${CFLAGS} -g -fsanitize=address" + if test $address -eq 1; then + CFLAGS="${CFLAGS} -g -fsanitize=address -fno-omit-frame-pointer" + fi + if test $memory -eq 1; then + CFLAGS="${CFLAGS} -g -fsanitize=memory -fno-omit-frame-pointer" fi if test $debug -eq 1; then CFLAGS="${CFLAGS} -DZLIB_DEBUG" @@ -260,7 +265,9 @@ if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then SHAREDLIBV=libz.$VER$shared_ext SHAREDLIBM=libz.$VER1$shared_ext LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER3"} - if libtool -V 2>&1 | grep Apple > /dev/null; then + if "${CROSS_PREFIX}libtool" -V 2>&1 | grep Apple > /dev/null; then + AR="${CROSS_PREFIX}libtool" + elif libtool -V 2>&1 | grep Apple > /dev/null; then AR="libtool" else AR="/usr/bin/libtool" @@ -435,7 +442,7 @@ EOF if test $shared -eq 1; then echo Checking for shared library support... | tee -a configure.log # we must test in two steps (cc then ld), required at least on SunOS 4.x - if try $CC -w -c $SFLAGS $test.c && + if try $CC -c $SFLAGS $test.c && try $LDSHARED $SFLAGS -o $test$shared_ext $test.o; then echo Building shared library $SHAREDLIBV with $CC. | tee -a configure.log elif test -z "$old_cc" -a -z "$old_cflags"; then @@ -860,7 +867,7 @@ echo prefix = $prefix >> configure.log echo sharedlibdir = $sharedlibdir >> configure.log echo uname = $uname >> configure.log -# udpate Makefile with the configure results +# update Makefile with the configure results sed < ${SRCDIR}Makefile.in " /^CC *=/s#=.*#=$CC# /^CFLAGS *=/s#=.*#=$CFLAGS# diff --git a/src/native/external/zlib/crc32.c b/src/native/external/zlib/crc32.c index f8357b083f7..6c38f5c04c6 100644 --- a/src/native/external/zlib/crc32.c +++ b/src/native/external/zlib/crc32.c @@ -103,19 +103,6 @@ # define ARMCRC32 #endif -/* Local functions. */ -local z_crc_t multmodp OF((z_crc_t a, z_crc_t b)); -local z_crc_t x2nmodp OF((z_off64_t n, unsigned k)); - -#if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) - local z_word_t byte_swap OF((z_word_t word)); -#endif - -#if defined(W) && !defined(ARMCRC32) - local z_crc_t crc_word OF((z_word_t data)); - local z_word_t crc_word_big OF((z_word_t data)); -#endif - #if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) /* Swap the bytes in a z_word_t to convert between little and big endian. Any @@ -123,9 +110,7 @@ local z_crc_t x2nmodp OF((z_off64_t n, unsigned k)); instruction, if one is available. This assumes that word_t is either 32 bits or 64 bits. */ -local z_word_t byte_swap(word) - z_word_t word; -{ +local z_word_t byte_swap(z_word_t word) { # if W == 8 return (word & 0xff00000000000000) >> 56 | @@ -146,24 +131,77 @@ local z_word_t byte_swap(word) } #endif +#ifdef DYNAMIC_CRC_TABLE +/* ========================================================================= + * Table of powers of x for combining CRC-32s, filled in by make_crc_table() + * below. + */ + local z_crc_t FAR x2n_table[32]; +#else +/* ========================================================================= + * Tables for byte-wise and braided CRC-32 calculations, and a table of powers + * of x for combining CRC-32s, all made by make_crc_table(). + */ +# include "crc32.h" +#endif + /* CRC polynomial. */ #define POLY 0xedb88320 /* p(x) reflected, with x^32 implied */ -#ifdef DYNAMIC_CRC_TABLE +/* + Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, + reflected. For speed, this requires that a not be zero. + */ +local z_crc_t multmodp(z_crc_t a, z_crc_t b) { + z_crc_t m, p; + + m = (z_crc_t)1 << 31; + p = 0; + for (;;) { + if (a & m) { + p ^= b; + if ((a & (m - 1)) == 0) + break; + } + m >>= 1; + b = b & 1 ? (b >> 1) ^ POLY : b >> 1; + } + return p; +} +/* + Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been + initialized. + */ +local z_crc_t x2nmodp(z_off64_t n, unsigned k) { + z_crc_t p; + + p = (z_crc_t)1 << 31; /* x^0 == 1 */ + while (n) { + if (n & 1) + p = multmodp(x2n_table[k & 31], p); + n >>= 1; + k++; + } + return p; +} + +#ifdef DYNAMIC_CRC_TABLE +/* ========================================================================= + * Build the tables for byte-wise and braided CRC-32 calculations, and a table + * of powers of x for combining CRC-32s. + */ local z_crc_t FAR crc_table[256]; -local z_crc_t FAR x2n_table[32]; -local void make_crc_table OF((void)); #ifdef W local z_word_t FAR crc_big_table[256]; local z_crc_t FAR crc_braid_table[W][256]; local z_word_t FAR crc_braid_big_table[W][256]; - local void braid OF((z_crc_t [][256], z_word_t [][256], int, int)); + local void braid(z_crc_t [][256], z_word_t [][256], int, int); #endif #ifdef MAKECRCH - local void write_table OF((FILE *, const z_crc_t FAR *, int)); - local void write_table32hi OF((FILE *, const z_word_t FAR *, int)); - local void write_table64 OF((FILE *, const z_word_t FAR *, int)); + local void write_table(FILE *, const z_crc_t FAR *, int); + local void write_table32hi(FILE *, const z_word_t FAR *, int); + local void write_table64(FILE *, const z_word_t FAR *, int); #endif /* MAKECRCH */ /* @@ -176,7 +214,6 @@ local void make_crc_table OF((void)); /* Definition of once functionality. */ typedef struct once_s once_t; -local void once OF((once_t *, void (*)(void))); /* Check for the availability of atomics. */ #if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \ @@ -196,10 +233,7 @@ struct once_s { invoke once() at the same time. The state must be a once_t initialized with ONCE_INIT. */ -local void once(state, init) - once_t *state; - void (*init)(void); -{ +local void once(once_t *state, void (*init)(void)) { if (!atomic_load(&state->done)) { if (atomic_flag_test_and_set(&state->begun)) while (!atomic_load(&state->done)) @@ -222,10 +256,7 @@ struct once_s { /* Test and set. Alas, not atomic, but tries to minimize the period of vulnerability. */ -local int test_and_set OF((int volatile *)); -local int test_and_set(flag) - int volatile *flag; -{ +local int test_and_set(int volatile *flag) { int was; was = *flag; @@ -234,10 +265,7 @@ local int test_and_set(flag) } /* Run the provided init() function once. This is not thread-safe. */ -local void once(state, init) - once_t *state; - void (*init)(void); -{ +local void once(once_t *state, void (*init)(void)) { if (!state->done) { if (test_and_set(&state->begun)) while (!state->done) @@ -279,8 +307,7 @@ local once_t made = ONCE_INIT; combinations of CRC register values and incoming bytes. */ -local void make_crc_table() -{ +local void make_crc_table(void) { unsigned i, j, n; z_crc_t p; @@ -447,11 +474,7 @@ local void make_crc_table() Write the 32-bit values in table[0..k-1] to out, five per line in hexadecimal separated by commas. */ -local void write_table(out, table, k) - FILE *out; - const z_crc_t FAR *table; - int k; -{ +local void write_table(FILE *out, const z_crc_t FAR *table, int k) { int n; for (n = 0; n < k; n++) @@ -464,11 +487,7 @@ local void write_table(out, table, k) Write the high 32-bits of each value in table[0..k-1] to out, five per line in hexadecimal separated by commas. */ -local void write_table32hi(out, table, k) -FILE *out; -const z_word_t FAR *table; -int k; -{ +local void write_table32hi(FILE *out, const z_word_t FAR *table, int k) { int n; for (n = 0; n < k; n++) @@ -484,11 +503,7 @@ int k; bits. If not, then the type cast and format string can be adjusted accordingly. */ -local void write_table64(out, table, k) - FILE *out; - const z_word_t FAR *table; - int k; -{ +local void write_table64(FILE *out, const z_word_t FAR *table, int k) { int n; for (n = 0; n < k; n++) @@ -498,8 +513,7 @@ local void write_table64(out, table, k) } /* Actually do the deed. */ -int main() -{ +int main(void) { make_crc_table(); return 0; } @@ -511,12 +525,7 @@ int main() Generate the little and big-endian braid tables for the given n and z_word_t size w. Each array must have room for w blocks of 256 elements. */ -local void braid(ltl, big, n, w) - z_crc_t ltl[][256]; - z_word_t big[][256]; - int n; - int w; -{ +local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { int k; z_crc_t i, p, q; for (k = 0; k < w; k++) { @@ -531,69 +540,13 @@ local void braid(ltl, big, n, w) } #endif -#else /* !DYNAMIC_CRC_TABLE */ -/* ======================================================================== - * Tables for byte-wise and braided CRC-32 calculations, and a table of powers - * of x for combining CRC-32s, all made by make_crc_table(). - */ -#include "crc32.h" #endif /* DYNAMIC_CRC_TABLE */ -/* ======================================================================== - * Routines used for CRC calculation. Some are also required for the table - * generation above. - */ - -/* - Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, - reflected. For speed, this requires that a not be zero. - */ -local z_crc_t multmodp(a, b) - z_crc_t a; - z_crc_t b; -{ - z_crc_t m, p; - - m = (z_crc_t)1 << 31; - p = 0; - for (;;) { - if (a & m) { - p ^= b; - if ((a & (m - 1)) == 0) - break; - } - m >>= 1; - b = b & 1 ? (b >> 1) ^ POLY : b >> 1; - } - return p; -} - -/* - Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been - initialized. - */ -local z_crc_t x2nmodp(n, k) - z_off64_t n; - unsigned k; -{ - z_crc_t p; - - p = (z_crc_t)1 << 31; /* x^0 == 1 */ - while (n) { - if (n & 1) - p = multmodp(x2n_table[k & 31], p); - n >>= 1; - k++; - } - return p; -} - /* ========================================================================= * This function can be used by asm versions of crc32(), and to force the * generation of the CRC tables in a threaded application. */ -const z_crc_t FAR * ZEXPORT get_crc_table() -{ +const z_crc_t FAR * ZEXPORT get_crc_table(void) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -619,11 +572,8 @@ const z_crc_t FAR * ZEXPORT get_crc_table() #define Z_BATCH_ZEROS 0xa10d3d0c /* computed from Z_BATCH = 3990 */ #define Z_BATCH_MIN 800 /* fewest words in a final batch */ -unsigned long ZEXPORT crc32_z(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - z_size_t len; -{ +unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, + z_size_t len) { z_crc_t val; z_word_t crc1, crc2; const z_word_t *word; @@ -723,18 +673,14 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) least-significant byte of the word as the first byte of data, without any pre or post conditioning. This is used to combine the CRCs of each braid. */ -local z_crc_t crc_word(data) - z_word_t data; -{ +local z_crc_t crc_word(z_word_t data) { int k; for (k = 0; k < W; k++) data = (data >> 8) ^ crc_table[data & 0xff]; return (z_crc_t)data; } -local z_word_t crc_word_big(data) - z_word_t data; -{ +local z_word_t crc_word_big(z_word_t data) { int k; for (k = 0; k < W; k++) data = (data << 8) ^ @@ -745,11 +691,8 @@ local z_word_t crc_word_big(data) #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32_z(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - z_size_t len; -{ +unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, + z_size_t len) { /* Return initial CRC, if requested. */ if (buf == Z_NULL) return 0; @@ -781,8 +724,8 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) words = (z_word_t const *)buf; /* Do endian check at execution time instead of compile time, since ARM - processors can change the endianess at execution time. If the - compiler knows what the endianess will be, it can optimize out the + processors can change the endianness at execution time. If the + compiler knows what the endianness will be, it can optimize out the check and the unused branch. */ endian = 1; if (*(unsigned char *)&endian) { @@ -1069,20 +1012,13 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32(crc, buf, len) - unsigned long crc; - const unsigned char FAR *buf; - uInt len; -{ +unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf, + uInt len) { return crc32_z(crc, buf, len); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine64(crc1, crc2, len2) - uLong crc1; - uLong crc2; - z_off64_t len2; -{ +uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -1090,18 +1026,12 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) } /* ========================================================================= */ -uLong ZEXPORT crc32_combine(crc1, crc2, len2) - uLong crc1; - uLong crc2; - z_off_t len2; -{ +uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) { return crc32_combine64(crc1, crc2, (z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen64(len2) - z_off64_t len2; -{ +uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -1109,17 +1039,11 @@ uLong ZEXPORT crc32_combine_gen64(len2) } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen(len2) - z_off_t len2; -{ +uLong ZEXPORT crc32_combine_gen(z_off_t len2) { return crc32_combine_gen64((z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_op(crc1, crc2, op) - uLong crc1; - uLong crc2; - uLong op; -{ +uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op) { return multmodp(op, crc1) ^ (crc2 & 0xffffffff); } diff --git a/src/native/external/zlib/deflate.c b/src/native/external/zlib/deflate.c index b7636639754..ca2fc59a1b5 100644 --- a/src/native/external/zlib/deflate.c +++ b/src/native/external/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -52,7 +52,7 @@ #include "deflate.h" const char deflate_copyright[] = - " deflate 1.2.13 Copyright 1995-2022 Jean-loup Gailly and Mark Adler "; + " deflate 1.3.1 Copyright 1995-2024 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -60,9 +60,6 @@ const char deflate_copyright[] = copyright string in the executable of your product. */ -/* =========================================================================== - * Function prototypes. - */ typedef enum { need_more, /* block not completed, need more input or more output */ block_done, /* block flush performed */ @@ -70,29 +67,16 @@ typedef enum { finish_done /* finish done, accept no more input or output */ } block_state; -typedef block_state (*compress_func) OF((deflate_state *s, int flush)); +typedef block_state (*compress_func)(deflate_state *s, int flush); /* Compression function. Returns the block state after the call. */ -local int deflateStateCheck OF((z_streamp strm)); -local void slide_hash OF((deflate_state *s)); -local void fill_window OF((deflate_state *s)); -local block_state deflate_stored OF((deflate_state *s, int flush)); -local block_state deflate_fast OF((deflate_state *s, int flush)); +local block_state deflate_stored(deflate_state *s, int flush); +local block_state deflate_fast(deflate_state *s, int flush); #ifndef FASTEST -local block_state deflate_slow OF((deflate_state *s, int flush)); -#endif -local block_state deflate_rle OF((deflate_state *s, int flush)); -local block_state deflate_huff OF((deflate_state *s, int flush)); -local void lm_init OF((deflate_state *s)); -local void putShortMSB OF((deflate_state *s, uInt b)); -local void flush_pending OF((z_streamp strm)); -local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); -local uInt longest_match OF((deflate_state *s, IPos cur_match)); - -#ifdef ZLIB_DEBUG -local void check_match OF((deflate_state *s, IPos start, IPos match, - int length)); +local block_state deflate_slow(deflate_state *s, int flush); #endif +local block_state deflate_rle(deflate_state *s, int flush); +local block_state deflate_huff(deflate_state *s, int flush); /* =========================================================================== * Local data @@ -195,9 +179,12 @@ local const config configuration_table[10] = { * bit values at the expense of memory usage). We slide even when level == 0 to * keep the hash table consistent if we switch back to level > 0 later. */ -local void slide_hash(s) - deflate_state *s; -{ +#if defined(__has_feature) +# if __has_feature(memory_sanitizer) + __attribute__((no_sanitize("memory"))) +# endif +#endif +local void slide_hash(deflate_state *s) { unsigned n, m; Posf *p; uInt wsize = s->w_size; @@ -221,30 +208,177 @@ local void slide_hash(s) #endif } +/* =========================================================================== + * Read a new buffer from the current input stream, update the adler32 + * and total number of bytes read. All deflate() input goes through + * this function so some applications may wish to modify it to avoid + * allocating a large strm->next_in buffer and copying from it. + * (See also flush_pending()). + */ +local unsigned read_buf(z_streamp strm, Bytef *buf, unsigned size) { + unsigned len = strm->avail_in; + + if (len > size) len = size; + if (len == 0) return 0; + + strm->avail_in -= len; + + zmemcpy(buf, strm->next_in, len); + if (strm->state->wrap == 1) { + strm->adler = adler32(strm->adler, buf, len); + } +#ifdef GZIP + else if (strm->state->wrap == 2) { + strm->adler = crc32(strm->adler, buf, len); + } +#endif + strm->next_in += len; + strm->total_in += len; + + return len; +} + +/* =========================================================================== + * Fill the window when the lookahead becomes insufficient. + * Updates strstart and lookahead. + * + * IN assertion: lookahead < MIN_LOOKAHEAD + * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD + * At least one byte has been read, or avail_in == 0; reads are + * performed for at least two bytes (required for the zip translate_eol + * option -- not supported here). + */ +local void fill_window(deflate_state *s) { + unsigned n; + unsigned more; /* Amount of free space at the end of the window. */ + uInt wsize = s->w_size; + + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + + do { + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); + + /* Deal with !@#$% 64K limit: */ + if (sizeof(int) <= 2) { + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { + more = wsize; + + } else if (more == (unsigned)(-1)) { + /* Very unlikely, but possible on 16 bit machine if + * strstart == 0 && lookahead == 1 (input done a byte at time) + */ + more--; + } + } + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (s->strstart >= wsize + MAX_DIST(s)) { + + zmemcpy(s->window, s->window + wsize, (unsigned)wsize - more); + s->match_start -= wsize; + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ + s->block_start -= (long) wsize; + if (s->insert > s->strstart) + s->insert = s->strstart; + slide_hash(s); + more += wsize; + } + if (s->strm->avail_in == 0) break; + + /* If there was no sliding: + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && + * more == window_size - lookahead - strstart + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) + * => more >= window_size - 2*WSIZE + 2 + * In the BIG_MEM or MMAP case (not yet supported), + * window_size == input_size + MIN_LOOKAHEAD && + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. + * Otherwise, window_size == 2*WSIZE so more >= 2. + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. + */ + Assert(more >= 2, "more < 2"); + + n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); + s->lookahead += n; + + /* Initialize the hash value now that we have some input: */ + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } + } + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, + * but this is not important since only literal bytes will be emitted. + */ + + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); +} + /* ========================================================================= */ -int ZEXPORT deflateInit_(strm, level, version, stream_size) - z_streamp strm; - int level; - const char *version; - int stream_size; -{ +int ZEXPORT deflateInit_(z_streamp strm, int level, const char *version, + int stream_size) { return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, version, stream_size); /* To do: ignore strm->next_in if we use it as window */ } /* ========================================================================= */ -int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, - version, stream_size) - z_streamp strm; - int level; - int method; - int windowBits; - int memLevel; - int strategy; - const char *version; - int stream_size; -{ +int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, + int windowBits, int memLevel, int strategy, + const char *version, int stream_size) { deflate_state *s; int wrap = 1; static const char my_version[] = ZLIB_VERSION; @@ -359,7 +493,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, * symbols from which it is being constructed. */ - s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, LIT_BUFS); s->pending_buf_size = (ulg)s->lit_bufsize * 4; if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || @@ -369,8 +503,14 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, deflateEnd (strm); return Z_MEM_ERROR; } +#ifdef LIT_MEM + s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1)); + s->l_buf = s->pending_buf + (s->lit_bufsize << 2); + s->sym_end = s->lit_bufsize - 1; +#else s->sym_buf = s->pending_buf + s->lit_bufsize; s->sym_end = (s->lit_bufsize - 1) * 3; +#endif /* We avoid equality with lit_bufsize*3 because of wraparound at 64K * on 16 bit machines and because stored blocks are restricted to * 64K-1 bytes. @@ -386,9 +526,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, /* ========================================================================= * Check for a valid deflate stream state. Return 0 if ok, 1 if not. */ -local int deflateStateCheck(strm) - z_streamp strm; -{ +local int deflateStateCheck(z_streamp strm) { deflate_state *s; if (strm == Z_NULL || strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) @@ -409,11 +547,8 @@ local int deflateStateCheck(strm) } /* ========================================================================= */ -int ZEXPORT deflateSetDictionary(strm, dictionary, dictLength) - z_streamp strm; - const Bytef *dictionary; - uInt dictLength; -{ +int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary, + uInt dictLength) { deflate_state *s; uInt str, n; int wrap; @@ -478,11 +613,8 @@ int ZEXPORT deflateSetDictionary(strm, dictionary, dictLength) } /* ========================================================================= */ -int ZEXPORT deflateGetDictionary(strm, dictionary, dictLength) - z_streamp strm; - Bytef *dictionary; - uInt *dictLength; -{ +int ZEXPORT deflateGetDictionary(z_streamp strm, Bytef *dictionary, + uInt *dictLength) { deflate_state *s; uInt len; @@ -500,9 +632,7 @@ int ZEXPORT deflateGetDictionary(strm, dictionary, dictLength) } /* ========================================================================= */ -int ZEXPORT deflateResetKeep(strm) - z_streamp strm; -{ +int ZEXPORT deflateResetKeep(z_streamp strm) { deflate_state *s; if (deflateStateCheck(strm)) { @@ -537,10 +667,32 @@ int ZEXPORT deflateResetKeep(strm) return Z_OK; } +/* =========================================================================== + * Initialize the "longest match" routines for a new zlib stream + */ +local void lm_init(deflate_state *s) { + s->window_size = (ulg)2L*s->w_size; + + CLEAR_HASH(s); + + /* Set the default configuration parameters: + */ + s->max_lazy_match = configuration_table[s->level].max_lazy; + s->good_match = configuration_table[s->level].good_length; + s->nice_match = configuration_table[s->level].nice_length; + s->max_chain_length = configuration_table[s->level].max_chain; + + s->strstart = 0; + s->block_start = 0L; + s->lookahead = 0; + s->insert = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + s->ins_h = 0; +} + /* ========================================================================= */ -int ZEXPORT deflateReset(strm) - z_streamp strm; -{ +int ZEXPORT deflateReset(z_streamp strm) { int ret; ret = deflateResetKeep(strm); @@ -550,10 +702,7 @@ int ZEXPORT deflateReset(strm) } /* ========================================================================= */ -int ZEXPORT deflateSetHeader(strm, head) - z_streamp strm; - gz_headerp head; -{ +int ZEXPORT deflateSetHeader(z_streamp strm, gz_headerp head) { if (deflateStateCheck(strm) || strm->state->wrap != 2) return Z_STREAM_ERROR; strm->state->gzhead = head; @@ -561,11 +710,7 @@ int ZEXPORT deflateSetHeader(strm, head) } /* ========================================================================= */ -int ZEXPORT deflatePending(strm, pending, bits) - unsigned *pending; - int *bits; - z_streamp strm; -{ +int ZEXPORT deflatePending(z_streamp strm, unsigned *pending, int *bits) { if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; @@ -575,19 +720,21 @@ int ZEXPORT deflatePending(strm, pending, bits) } /* ========================================================================= */ -int ZEXPORT deflatePrime(strm, bits, value) - z_streamp strm; - int bits; - int value; -{ +int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { deflate_state *s; int put; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; +#ifdef LIT_MEM + if (bits < 0 || bits > 16 || + (uchf *)s->d_buf < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; +#else if (bits < 0 || bits > 16 || s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; +#endif do { put = Buf_size - s->bi_valid; if (put > bits) @@ -602,11 +749,7 @@ int ZEXPORT deflatePrime(strm, bits, value) } /* ========================================================================= */ -int ZEXPORT deflateParams(strm, level, strategy) - z_streamp strm; - int level; - int strategy; -{ +int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) { deflate_state *s; compress_func func; @@ -651,13 +794,8 @@ int ZEXPORT deflateParams(strm, level, strategy) } /* ========================================================================= */ -int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) - z_streamp strm; - int good_length; - int max_lazy; - int nice_length; - int max_chain; -{ +int ZEXPORT deflateTune(z_streamp strm, int good_length, int max_lazy, + int nice_length, int max_chain) { deflate_state *s; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -693,10 +831,7 @@ int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) * * Shifts are used to approximate divisions, for speed. */ -uLong ZEXPORT deflateBound(strm, sourceLen) - z_streamp strm; - uLong sourceLen; -{ +uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { deflate_state *s; uLong fixedlen, storelen, wraplen; @@ -766,10 +901,7 @@ uLong ZEXPORT deflateBound(strm, sourceLen) * IN assertion: the stream state is correct and there is enough room in * pending_buf. */ -local void putShortMSB(s, b) - deflate_state *s; - uInt b; -{ +local void putShortMSB(deflate_state *s, uInt b) { put_byte(s, (Byte)(b >> 8)); put_byte(s, (Byte)(b & 0xff)); } @@ -780,9 +912,7 @@ local void putShortMSB(s, b) * applications may wish to modify it to avoid allocating a large * strm->next_out buffer and copying into it. (See also read_buf()). */ -local void flush_pending(strm) - z_streamp strm; -{ +local void flush_pending(z_streamp strm) { unsigned len; deflate_state *s = strm->state; @@ -813,10 +943,7 @@ local void flush_pending(strm) } while (0) /* ========================================================================= */ -int ZEXPORT deflate(strm, flush) - z_streamp strm; - int flush; -{ +int ZEXPORT deflate(z_streamp strm, int flush) { int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; @@ -1128,9 +1255,7 @@ int ZEXPORT deflate(strm, flush) } /* ========================================================================= */ -int ZEXPORT deflateEnd(strm) - z_streamp strm; -{ +int ZEXPORT deflateEnd(z_streamp strm) { int status; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1154,11 +1279,10 @@ int ZEXPORT deflateEnd(strm) * To simplify the source, this is not supported for 16-bit MSDOS (which * doesn't have enough memory anyway to duplicate compression states). */ -int ZEXPORT deflateCopy(dest, source) - z_streamp dest; - z_streamp source; -{ +int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { #ifdef MAXSEG_64K + (void)dest; + (void)source; return Z_STREAM_ERROR; #else deflate_state *ds; @@ -1182,7 +1306,7 @@ int ZEXPORT deflateCopy(dest, source) ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); - ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4); + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS); if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || ds->pending_buf == Z_NULL) { @@ -1193,10 +1317,15 @@ int ZEXPORT deflateCopy(dest, source) zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); - zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); + zmemcpy(ds->pending_buf, ss->pending_buf, ds->lit_bufsize * LIT_BUFS); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); +#ifdef LIT_MEM + ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1)); + ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2); +#else ds->sym_buf = ds->pending_buf + ds->lit_bufsize; +#endif ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; @@ -1206,66 +1335,6 @@ int ZEXPORT deflateCopy(dest, source) #endif /* MAXSEG_64K */ } -/* =========================================================================== - * Read a new buffer from the current input stream, update the adler32 - * and total number of bytes read. All deflate() input goes through - * this function so some applications may wish to modify it to avoid - * allocating a large strm->next_in buffer and copying from it. - * (See also flush_pending()). - */ -local unsigned read_buf(strm, buf, size) - z_streamp strm; - Bytef *buf; - unsigned size; -{ - unsigned len = strm->avail_in; - - if (len > size) len = size; - if (len == 0) return 0; - - strm->avail_in -= len; - - zmemcpy(buf, strm->next_in, len); - if (strm->state->wrap == 1) { - strm->adler = adler32(strm->adler, buf, len); - } -#ifdef GZIP - else if (strm->state->wrap == 2) { - strm->adler = crc32(strm->adler, buf, len); - } -#endif - strm->next_in += len; - strm->total_in += len; - - return len; -} - -/* =========================================================================== - * Initialize the "longest match" routines for a new zlib stream - */ -local void lm_init(s) - deflate_state *s; -{ - s->window_size = (ulg)2L*s->w_size; - - CLEAR_HASH(s); - - /* Set the default configuration parameters: - */ - s->max_lazy_match = configuration_table[s->level].max_lazy; - s->good_match = configuration_table[s->level].good_length; - s->nice_match = configuration_table[s->level].nice_length; - s->max_chain_length = configuration_table[s->level].max_chain; - - s->strstart = 0; - s->block_start = 0L; - s->lookahead = 0; - s->insert = 0; - s->match_length = s->prev_length = MIN_MATCH-1; - s->match_available = 0; - s->ins_h = 0; -} - #ifndef FASTEST /* =========================================================================== * Set match_start to the longest match starting at the given string and @@ -1276,10 +1345,7 @@ local void lm_init(s) * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 * OUT assertion: the match length is not greater than s->lookahead. */ -local uInt longest_match(s, cur_match) - deflate_state *s; - IPos cur_match; /* current match */ -{ +local uInt longest_match(deflate_state *s, IPos cur_match) { unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ @@ -1427,10 +1493,7 @@ local uInt longest_match(s, cur_match) /* --------------------------------------------------------------------------- * Optimized version for FASTEST only */ -local uInt longest_match(s, cur_match) - deflate_state *s; - IPos cur_match; /* current match */ -{ +local uInt longest_match(deflate_state *s, IPos cur_match) { register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ @@ -1491,19 +1554,23 @@ local uInt longest_match(s, cur_match) /* =========================================================================== * Check that the match at match_start is indeed a match. */ -local void check_match(s, start, match, length) - deflate_state *s; - IPos start, match; - int length; -{ +local void check_match(deflate_state *s, IPos start, IPos match, int length) { /* check that the match is indeed a match */ - if (zmemcmp(s->window + match, - s->window + start, length) != EQUAL) { - fprintf(stderr, " start %u, match %u, length %d\n", - start, match, length); + Bytef *back = s->window + (int)match, *here = s->window + start; + IPos len = length; + if (match == (IPos)-1) { + /* match starts one byte before the current window -- just compare the + subsequent length-1 bytes */ + back++; + here++; + len--; + } + if (zmemcmp(back, here, len) != EQUAL) { + fprintf(stderr, " start %u, match %d, length %d\n", + start, (int)match, length); do { - fprintf(stderr, "%c%c", s->window[match++], s->window[start++]); - } while (--length != 0); + fprintf(stderr, "(%02x %02x)", *back++, *here++); + } while (--len != 0); z_error("invalid match"); } if (z_verbose > 1) { @@ -1515,137 +1582,6 @@ local void check_match(s, start, match, length) # define check_match(s, start, match, length) #endif /* ZLIB_DEBUG */ -/* =========================================================================== - * Fill the window when the lookahead becomes insufficient. - * Updates strstart and lookahead. - * - * IN assertion: lookahead < MIN_LOOKAHEAD - * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD - * At least one byte has been read, or avail_in == 0; reads are - * performed for at least two bytes (required for the zip translate_eol - * option -- not supported here). - */ -local void fill_window(s) - deflate_state *s; -{ - unsigned n; - unsigned more; /* Amount of free space at the end of the window. */ - uInt wsize = s->w_size; - - Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); - - do { - more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); - - /* Deal with !@#$% 64K limit: */ - if (sizeof(int) <= 2) { - if (more == 0 && s->strstart == 0 && s->lookahead == 0) { - more = wsize; - - } else if (more == (unsigned)(-1)) { - /* Very unlikely, but possible on 16 bit machine if - * strstart == 0 && lookahead == 1 (input done a byte at time) - */ - more--; - } - } - - /* If the window is almost full and there is insufficient lookahead, - * move the upper half to the lower one to make room in the upper half. - */ - if (s->strstart >= wsize + MAX_DIST(s)) { - - zmemcpy(s->window, s->window + wsize, (unsigned)wsize - more); - s->match_start -= wsize; - s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ - s->block_start -= (long) wsize; - if (s->insert > s->strstart) - s->insert = s->strstart; - slide_hash(s); - more += wsize; - } - if (s->strm->avail_in == 0) break; - - /* If there was no sliding: - * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && - * more == window_size - lookahead - strstart - * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) - * => more >= window_size - 2*WSIZE + 2 - * In the BIG_MEM or MMAP case (not yet supported), - * window_size == input_size + MIN_LOOKAHEAD && - * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. - * Otherwise, window_size == 2*WSIZE so more >= 2. - * If there was sliding, more >= WSIZE. So in all cases, more >= 2. - */ - Assert(more >= 2, "more < 2"); - - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); - s->lookahead += n; - - /* Initialize the hash value now that we have some input: */ - if (s->lookahead + s->insert >= MIN_MATCH) { - uInt str = s->strstart - s->insert; - s->ins_h = s->window[str]; - UPDATE_HASH(s, s->ins_h, s->window[str + 1]); -#if MIN_MATCH != 3 - Call UPDATE_HASH() MIN_MATCH-3 more times -#endif - while (s->insert) { - UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); -#ifndef FASTEST - s->prev[str & s->w_mask] = s->head[s->ins_h]; -#endif - s->head[s->ins_h] = (Pos)str; - str++; - s->insert--; - if (s->lookahead + s->insert < MIN_MATCH) - break; - } - } - /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, - * but this is not important since only literal bytes will be emitted. - */ - - } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); - - /* If the WIN_INIT bytes after the end of the current data have never been - * written, then zero those bytes in order to avoid memory check reports of - * the use of uninitialized (or uninitialised as Julian writes) bytes by - * the longest match routines. Update the high water mark for the next - * time through here. WIN_INIT is set to MAX_MATCH since the longest match - * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. - */ - if (s->high_water < s->window_size) { - ulg curr = s->strstart + (ulg)(s->lookahead); - ulg init; - - if (s->high_water < curr) { - /* Previous high water mark below current data -- zero WIN_INIT - * bytes or up to end of window, whichever is less. - */ - init = s->window_size - curr; - if (init > WIN_INIT) - init = WIN_INIT; - zmemzero(s->window + curr, (unsigned)init); - s->high_water = curr + init; - } - else if (s->high_water < (ulg)curr + WIN_INIT) { - /* High water mark at or above current data, but below current data - * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up - * to end of window, whichever is less. - */ - init = (ulg)curr + WIN_INIT - s->high_water; - if (init > s->window_size - s->high_water) - init = s->window_size - s->high_water; - zmemzero(s->window + s->high_water, (unsigned)init); - s->high_water += init; - } - } - - Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, - "not enough room for search"); -} - /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. @@ -1688,10 +1624,7 @@ local void fill_window(s) * copied. It is most efficient with large input and output buffers, which * maximizes the opportunities to have a single copy from next_in to next_out. */ -local block_state deflate_stored(s, flush) - deflate_state *s; - int flush; -{ +local block_state deflate_stored(deflate_state *s, int flush) { /* Smallest worthy block size when not flushing or finishing. By default * this is 32K. This can be as small as 507 bytes for memLevel == 1. For * large input and output buffers, the stored block size will be larger. @@ -1875,10 +1808,7 @@ local block_state deflate_stored(s, flush) * new strings in the dictionary only for unmatched strings or for short * matches. It is used only for the fast compression options. */ -local block_state deflate_fast(s, flush) - deflate_state *s; - int flush; -{ +local block_state deflate_fast(deflate_state *s, int flush) { IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ @@ -1977,10 +1907,7 @@ local block_state deflate_fast(s, flush) * evaluation for matches: a match is finally adopted only if there is * no better match at the next window position. */ -local block_state deflate_slow(s, flush) - deflate_state *s; - int flush; -{ +local block_state deflate_slow(deflate_state *s, int flush) { IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ @@ -2108,10 +2035,7 @@ local block_state deflate_slow(s, flush) * one. Do not maintain a hash table. (It will be regenerated if this run of * deflate switches away from Z_RLE.) */ -local block_state deflate_rle(s, flush) - deflate_state *s; - int flush; -{ +local block_state deflate_rle(deflate_state *s, int flush) { int bflush; /* set if current block must be flushed */ uInt prev; /* byte at distance one to match */ Bytef *scan, *strend; /* scan goes up to strend for length of run */ @@ -2182,10 +2106,7 @@ local block_state deflate_rle(s, flush) * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. * (It will be regenerated if this run of deflate switches away from Huffman.) */ -local block_state deflate_huff(s, flush) - deflate_state *s; - int flush; -{ +local block_state deflate_huff(deflate_state *s, int flush) { int bflush; /* set if current block must be flushed */ for (;;) { diff --git a/src/native/external/zlib/deflate.h b/src/native/external/zlib/deflate.h index 1a06cd5f25d..300c6ada62b 100644 --- a/src/native/external/zlib/deflate.h +++ b/src/native/external/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2018 Jean-loup Gailly + * Copyright (C) 1995-2024 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -23,6 +23,10 @@ # define GZIP #endif +/* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at + the cost of a larger memory footprint */ +/* #define LIT_MEM */ + /* =========================================================================== * Internal compression state. */ @@ -217,7 +221,14 @@ typedef struct internal_state { /* Depth of each subtree used as tie breaker for trees of equal frequency */ +#ifdef LIT_MEM +# define LIT_BUFS 5 + ushf *d_buf; /* buffer for distances */ + uchf *l_buf; /* buffer for literals/lengths */ +#else +# define LIT_BUFS 4 uchf *sym_buf; /* buffer for distances and literals/lengths */ +#endif uInt lit_bufsize; /* Size of match buffer for literals/lengths. There are 4 reasons for @@ -239,7 +250,7 @@ typedef struct internal_state { * - I can't count above 4 */ - uInt sym_next; /* running index in sym_buf */ + uInt sym_next; /* running index in symbol buffer */ uInt sym_end; /* symbol table full when sym_next reaches this */ ulg opt_len; /* bit length of current block with optimal trees */ @@ -291,14 +302,14 @@ typedef struct internal_state { memory checker errors from longest match routines */ /* in trees.c */ -void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); -int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); -void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, - ulg stored_len, int last)); -void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); -void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); -void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, - ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_init(deflate_state *s); +int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc); +void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, + ulg stored_len, int last); +void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s); +void ZLIB_INTERNAL _tr_align(deflate_state *s); +void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, + ulg stored_len, int last); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) @@ -318,6 +329,25 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, extern const uch ZLIB_INTERNAL _dist_code[]; #endif +#ifdef LIT_MEM +# define _tr_tally_lit(s, c, flush) \ + { uch cc = (c); \ + s->d_buf[s->sym_next] = 0; \ + s->l_buf[s->sym_next++] = cc; \ + s->dyn_ltree[cc].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +# define _tr_tally_dist(s, distance, length, flush) \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ + s->d_buf[s->sym_next] = dist; \ + s->l_buf[s->sym_next++] = len; \ + dist--; \ + s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ + s->dyn_dtree[d_code(dist)].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +#else # define _tr_tally_lit(s, c, flush) \ { uch cc = (c); \ s->sym_buf[s->sym_next++] = 0; \ @@ -337,6 +367,7 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, s->dyn_dtree[d_code(dist)].Freq++; \ flush = (s->sym_next == s->sym_end); \ } +#endif #else # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) # define _tr_tally_dist(s, distance, length, flush) \ diff --git a/src/native/external/zlib/gzclose.c b/src/native/external/zlib/gzclose.c index caeb99a3177..48d6a86f04b 100644 --- a/src/native/external/zlib/gzclose.c +++ b/src/native/external/zlib/gzclose.c @@ -8,9 +8,7 @@ /* gzclose() is in a separate file so that it is linked in only if it is used. That way the other gzclose functions can be used instead to avoid linking in unneeded compression or decompression routines. */ -int ZEXPORT gzclose(file) - gzFile file; -{ +int ZEXPORT gzclose(gzFile file) { #ifndef NO_GZCOMPRESS gz_statep state; diff --git a/src/native/external/zlib/gzguts.h b/src/native/external/zlib/gzguts.h index 57faf37165a..eba72085bb7 100644 --- a/src/native/external/zlib/gzguts.h +++ b/src/native/external/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004-2019 Mark Adler + * Copyright (C) 2004-2024 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,9 +7,8 @@ # ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE 1 # endif -# ifdef _FILE_OFFSET_BITS -# undef _FILE_OFFSET_BITS -# endif +# undef _FILE_OFFSET_BITS +# undef _TIME_BITS #endif #ifdef HAVE_HIDDEN @@ -119,8 +118,8 @@ /* gz* functions always use library allocation functions */ #ifndef STDC - extern voidp malloc OF((uInt size)); - extern void free OF((voidpf ptr)); + extern voidp malloc(uInt size); + extern void free(voidpf ptr); #endif /* get errno and strerror definition */ @@ -138,10 +137,10 @@ /* provide prototypes for these when building zlib without LFS */ #if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 - ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); - ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); - ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); - ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); + ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int); + ZEXTERN z_off64_t ZEXPORT gztell64(gzFile); + ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile); #endif /* default memLevel */ @@ -203,17 +202,13 @@ typedef struct { typedef gz_state FAR *gz_statep; /* shared functions */ -void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +void ZLIB_INTERNAL gz_error(gz_statep, int, const char *); #if defined UNDER_CE -char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +char ZLIB_INTERNAL *gz_strwinerror(DWORD error); #endif /* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t value -- needed when comparing unsigned to z_off64_t, which is signed (possible z_off64_t types off_t, off64_t, and long are all signed) */ -#ifdef INT_MAX -# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) -#else -unsigned ZLIB_INTERNAL gz_intmax OF((void)); -# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) -#endif +unsigned ZLIB_INTERNAL gz_intmax(void); +#define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) diff --git a/src/native/external/zlib/gzlib.c b/src/native/external/zlib/gzlib.c index 55da46a453f..983153cc8e4 100644 --- a/src/native/external/zlib/gzlib.c +++ b/src/native/external/zlib/gzlib.c @@ -1,5 +1,5 @@ /* gzlib.c -- zlib functions common to reading and writing gzip files - * Copyright (C) 2004-2019 Mark Adler + * Copyright (C) 2004-2024 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -15,10 +15,6 @@ #endif #endif -/* Local functions */ -local void gz_reset OF((gz_statep)); -local gzFile gz_open OF((const void *, int, const char *)); - #if defined UNDER_CE /* Map the Windows error number in ERROR to a locale-dependent error message @@ -30,9 +26,7 @@ local gzFile gz_open OF((const void *, int, const char *)); The gz_strwinerror function does not change the current setting of GetLastError. */ -char ZLIB_INTERNAL *gz_strwinerror(error) - DWORD error; -{ +char ZLIB_INTERNAL *gz_strwinerror(DWORD error) { static char buf[1024]; wchar_t *msgbuf; @@ -72,9 +66,7 @@ char ZLIB_INTERNAL *gz_strwinerror(error) #endif /* UNDER_CE */ /* Reset gzip file state */ -local void gz_reset(state) - gz_statep state; -{ +local void gz_reset(gz_statep state) { state->x.have = 0; /* no output data available */ if (state->mode == GZ_READ) { /* for reading ... */ state->eof = 0; /* not at end of file */ @@ -90,11 +82,7 @@ local void gz_reset(state) } /* Open a gzip file either by name or file descriptor. */ -local gzFile gz_open(path, fd, mode) - const void *path; - int fd; - const char *mode; -{ +local gzFile gz_open(const void *path, int fd, const char *mode) { gz_statep state; z_size_t len; int oflag; @@ -269,26 +257,17 @@ local gzFile gz_open(path, fd, mode) } /* -- see zlib.h -- */ -gzFile ZEXPORT gzopen(path, mode) - const char *path; - const char *mode; -{ +gzFile ZEXPORT gzopen(const char *path, const char *mode) { return gz_open(path, -1, mode); } /* -- see zlib.h -- */ -gzFile ZEXPORT gzopen64(path, mode) - const char *path; - const char *mode; -{ +gzFile ZEXPORT gzopen64(const char *path, const char *mode) { return gz_open(path, -1, mode); } /* -- see zlib.h -- */ -gzFile ZEXPORT gzdopen(fd, mode) - int fd; - const char *mode; -{ +gzFile ZEXPORT gzdopen(int fd, const char *mode) { char *path; /* identifier for error messages */ gzFile gz; @@ -306,19 +285,13 @@ gzFile ZEXPORT gzdopen(fd, mode) /* -- see zlib.h -- */ #ifdef WIDECHAR -gzFile ZEXPORT gzopen_w(path, mode) - const wchar_t *path; - const char *mode; -{ +gzFile ZEXPORT gzopen_w(const wchar_t *path, const char *mode) { return gz_open(path, -2, mode); } #endif /* -- see zlib.h -- */ -int ZEXPORT gzbuffer(file, size) - gzFile file; - unsigned size; -{ +int ZEXPORT gzbuffer(gzFile file, unsigned size) { gz_statep state; /* get internal structure and check integrity */ @@ -335,16 +308,14 @@ int ZEXPORT gzbuffer(file, size) /* check and set requested size */ if ((size << 1) < size) return -1; /* need to be able to double it */ - if (size < 2) - size = 2; /* need two bytes to check magic header */ + if (size < 8) + size = 8; /* needed to behave well with flushing */ state->want = size; return 0; } /* -- see zlib.h -- */ -int ZEXPORT gzrewind(file) - gzFile file; -{ +int ZEXPORT gzrewind(gzFile file) { gz_statep state; /* get internal structure */ @@ -365,11 +336,7 @@ int ZEXPORT gzrewind(file) } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gzseek64(file, offset, whence) - gzFile file; - z_off64_t offset; - int whence; -{ +z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { unsigned n; z_off64_t ret; gz_statep state; @@ -442,11 +409,7 @@ z_off64_t ZEXPORT gzseek64(file, offset, whence) } /* -- see zlib.h -- */ -z_off_t ZEXPORT gzseek(file, offset, whence) - gzFile file; - z_off_t offset; - int whence; -{ +z_off_t ZEXPORT gzseek(gzFile file, z_off_t offset, int whence) { z_off64_t ret; ret = gzseek64(file, (z_off64_t)offset, whence); @@ -454,9 +417,7 @@ z_off_t ZEXPORT gzseek(file, offset, whence) } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gztell64(file) - gzFile file; -{ +z_off64_t ZEXPORT gztell64(gzFile file) { gz_statep state; /* get internal structure and check integrity */ @@ -471,9 +432,7 @@ z_off64_t ZEXPORT gztell64(file) } /* -- see zlib.h -- */ -z_off_t ZEXPORT gztell(file) - gzFile file; -{ +z_off_t ZEXPORT gztell(gzFile file) { z_off64_t ret; ret = gztell64(file); @@ -481,9 +440,7 @@ z_off_t ZEXPORT gztell(file) } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gzoffset64(file) - gzFile file; -{ +z_off64_t ZEXPORT gzoffset64(gzFile file) { z_off64_t offset; gz_statep state; @@ -504,9 +461,7 @@ z_off64_t ZEXPORT gzoffset64(file) } /* -- see zlib.h -- */ -z_off_t ZEXPORT gzoffset(file) - gzFile file; -{ +z_off_t ZEXPORT gzoffset(gzFile file) { z_off64_t ret; ret = gzoffset64(file); @@ -514,9 +469,7 @@ z_off_t ZEXPORT gzoffset(file) } /* -- see zlib.h -- */ -int ZEXPORT gzeof(file) - gzFile file; -{ +int ZEXPORT gzeof(gzFile file) { gz_statep state; /* get internal structure and check integrity */ @@ -531,10 +484,7 @@ int ZEXPORT gzeof(file) } /* -- see zlib.h -- */ -const char * ZEXPORT gzerror(file, errnum) - gzFile file; - int *errnum; -{ +const char * ZEXPORT gzerror(gzFile file, int *errnum) { gz_statep state; /* get internal structure and check integrity */ @@ -552,9 +502,7 @@ const char * ZEXPORT gzerror(file, errnum) } /* -- see zlib.h -- */ -void ZEXPORT gzclearerr(file) - gzFile file; -{ +void ZEXPORT gzclearerr(gzFile file) { gz_statep state; /* get internal structure and check integrity */ @@ -578,11 +526,7 @@ void ZEXPORT gzclearerr(file) memory). Simply save the error message as a static string. If there is an allocation failure constructing the error message, then convert the error to out of memory. */ -void ZLIB_INTERNAL gz_error(state, err, msg) - gz_statep state; - int err; - const char *msg; -{ +void ZLIB_INTERNAL gz_error(gz_statep state, int err, const char *msg) { /* free previously allocated message and clear */ if (state->msg != NULL) { if (state->err != Z_MEM_ERROR) @@ -619,21 +563,20 @@ void ZLIB_INTERNAL gz_error(state, err, msg) #endif } -#ifndef INT_MAX /* portably return maximum value for an int (when limits.h presumed not available) -- we need to do this to cover cases where 2's complement not used, since C standard permits 1's complement and sign-bit representations, otherwise we could just use ((unsigned)-1) >> 1 */ -unsigned ZLIB_INTERNAL gz_intmax() -{ - unsigned p, q; - - p = 1; +unsigned ZLIB_INTERNAL gz_intmax(void) { +#ifdef INT_MAX + return INT_MAX; +#else + unsigned p = 1, q; do { q = p; p <<= 1; p++; } while (p > q); return q >> 1; -} #endif +} diff --git a/src/native/external/zlib/gzread.c b/src/native/external/zlib/gzread.c index dd77381596c..4168cbc8875 100644 --- a/src/native/external/zlib/gzread.c +++ b/src/native/external/zlib/gzread.c @@ -5,25 +5,12 @@ #include "gzguts.h" -/* Local functions */ -local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); -local int gz_avail OF((gz_statep)); -local int gz_look OF((gz_statep)); -local int gz_decomp OF((gz_statep)); -local int gz_fetch OF((gz_statep)); -local int gz_skip OF((gz_statep, z_off64_t)); -local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); - /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. This function needs to loop on read(), since read() is not guaranteed to read the number of bytes requested, depending on the type of descriptor. */ -local int gz_load(state, buf, len, have) - gz_statep state; - unsigned char *buf; - unsigned len; - unsigned *have; -{ +local int gz_load(gz_statep state, unsigned char *buf, unsigned len, + unsigned *have) { int ret; unsigned get, max = ((unsigned)-1 >> 2) + 1; @@ -53,9 +40,7 @@ local int gz_load(state, buf, len, have) If strm->avail_in != 0, then the current data is moved to the beginning of the input buffer, and then the remainder of the buffer is loaded with the available data from the input file. */ -local int gz_avail(state) - gz_statep state; -{ +local int gz_avail(gz_statep state) { unsigned got; z_streamp strm = &(state->strm); @@ -88,9 +73,7 @@ local int gz_avail(state) case, all further file reads will be directly to either the output buffer or a user buffer. If decompressing, the inflate state will be initialized. gz_look() will return 0 on success or -1 on failure. */ -local int gz_look(state) - gz_statep state; -{ +local int gz_look(gz_statep state) { z_streamp strm = &(state->strm); /* allocate read buffers and inflate memory */ @@ -170,9 +153,7 @@ local int gz_look(state) data. If the gzip stream completes, state->how is reset to LOOK to look for the next gzip stream or raw data, once state->x.have is depleted. Returns 0 on success, -1 on failure. */ -local int gz_decomp(state) - gz_statep state; -{ +local int gz_decomp(gz_statep state) { int ret = Z_OK; unsigned had; z_streamp strm = &(state->strm); @@ -224,9 +205,7 @@ local int gz_decomp(state) looked for to determine whether to copy or decompress. Returns -1 on error, otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the end of the input file has been reached and all data has been processed. */ -local int gz_fetch(state) - gz_statep state; -{ +local int gz_fetch(gz_statep state) { z_streamp strm = &(state->strm); do { @@ -254,10 +233,7 @@ local int gz_fetch(state) } /* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ -local int gz_skip(state, len) - gz_statep state; - z_off64_t len; -{ +local int gz_skip(gz_statep state, z_off64_t len) { unsigned n; /* skip over len bytes or reach end-of-file, whichever comes first */ @@ -289,11 +265,7 @@ local int gz_skip(state, len) input. Return the number of bytes read. If zero is returned, either the end of file was reached, or there was an error. state->err must be consulted in that case to determine which. */ -local z_size_t gz_read(state, buf, len) - gz_statep state; - voidp buf; - z_size_t len; -{ +local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { z_size_t got; unsigned n; @@ -370,11 +342,7 @@ local z_size_t gz_read(state, buf, len) } /* -- see zlib.h -- */ -int ZEXPORT gzread(file, buf, len) - gzFile file; - voidp buf; - unsigned len; -{ +int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { gz_statep state; /* get internal structure */ @@ -406,12 +374,7 @@ int ZEXPORT gzread(file, buf, len) } /* -- see zlib.h -- */ -z_size_t ZEXPORT gzfread(buf, size, nitems, file) - voidp buf; - z_size_t size; - z_size_t nitems; - gzFile file; -{ +z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) { z_size_t len; gz_statep state; @@ -442,9 +405,7 @@ z_size_t ZEXPORT gzfread(buf, size, nitems, file) #else # undef gzgetc #endif -int ZEXPORT gzgetc(file) - gzFile file; -{ +int ZEXPORT gzgetc(gzFile file) { unsigned char buf[1]; gz_statep state; @@ -469,17 +430,12 @@ int ZEXPORT gzgetc(file) return gz_read(state, buf, 1) < 1 ? -1 : buf[0]; } -int ZEXPORT gzgetc_(file) -gzFile file; -{ +int ZEXPORT gzgetc_(gzFile file) { return gzgetc(file); } /* -- see zlib.h -- */ -int ZEXPORT gzungetc(c, file) - int c; - gzFile file; -{ +int ZEXPORT gzungetc(int c, gzFile file) { gz_statep state; /* get internal structure */ @@ -487,6 +443,10 @@ int ZEXPORT gzungetc(c, file) return -1; state = (gz_statep)file; + /* in case this was just opened, set up the input buffer */ + if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) + (void)gz_look(state); + /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) @@ -536,11 +496,7 @@ int ZEXPORT gzungetc(c, file) } /* -- see zlib.h -- */ -char * ZEXPORT gzgets(file, buf, len) - gzFile file; - char *buf; - int len; -{ +char * ZEXPORT gzgets(gzFile file, char *buf, int len) { unsigned left, n; char *str; unsigned char *eol; @@ -600,9 +556,7 @@ char * ZEXPORT gzgets(file, buf, len) } /* -- see zlib.h -- */ -int ZEXPORT gzdirect(file) - gzFile file; -{ +int ZEXPORT gzdirect(gzFile file) { gz_statep state; /* get internal structure */ @@ -620,9 +574,7 @@ int ZEXPORT gzdirect(file) } /* -- see zlib.h -- */ -int ZEXPORT gzclose_r(file) - gzFile file; -{ +int ZEXPORT gzclose_r(gzFile file) { int ret, err; gz_statep state; diff --git a/src/native/external/zlib/gzwrite.c b/src/native/external/zlib/gzwrite.c index eb8a0e5893f..435b4621b53 100644 --- a/src/native/external/zlib/gzwrite.c +++ b/src/native/external/zlib/gzwrite.c @@ -5,18 +5,10 @@ #include "gzguts.h" -/* Local functions */ -local int gz_init OF((gz_statep)); -local int gz_comp OF((gz_statep, int)); -local int gz_zero OF((gz_statep, z_off64_t)); -local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); - /* Initialize state for writing a gzip file. Mark initialization by setting state->size to non-zero. Return -1 on a memory allocation failure, or 0 on success. */ -local int gz_init(state) - gz_statep state; -{ +local int gz_init(gz_statep state) { int ret; z_streamp strm = &(state->strm); @@ -70,10 +62,7 @@ local int gz_init(state) deflate() flush value. If flush is Z_FINISH, then the deflate() state is reset to start a new gzip stream. If gz->direct is true, then simply write to the output file without compressing, and ignore flush. */ -local int gz_comp(state, flush) - gz_statep state; - int flush; -{ +local int gz_comp(gz_statep state, int flush) { int ret, writ; unsigned have, put, max = ((unsigned)-1 >> 2) + 1; z_streamp strm = &(state->strm); @@ -151,10 +140,7 @@ local int gz_comp(state, flush) /* Compress len zeros to output. Return -1 on a write error or memory allocation failure by gz_comp(), or 0 on success. */ -local int gz_zero(state, len) - gz_statep state; - z_off64_t len; -{ +local int gz_zero(gz_statep state, z_off64_t len) { int first; unsigned n; z_streamp strm = &(state->strm); @@ -184,11 +170,7 @@ local int gz_zero(state, len) /* Write len bytes from buf to file. Return the number of bytes written. If the returned value is less than len, then there was an error. */ -local z_size_t gz_write(state, buf, len) - gz_statep state; - voidpc buf; - z_size_t len; -{ +local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { z_size_t put = len; /* if len is zero, avoid unnecessary operations */ @@ -252,11 +234,7 @@ local z_size_t gz_write(state, buf, len) } /* -- see zlib.h -- */ -int ZEXPORT gzwrite(file, buf, len) - gzFile file; - voidpc buf; - unsigned len; -{ +int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { gz_statep state; /* get internal structure */ @@ -280,12 +258,8 @@ int ZEXPORT gzwrite(file, buf, len) } /* -- see zlib.h -- */ -z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) - voidpc buf; - z_size_t size; - z_size_t nitems; - gzFile file; -{ +z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, z_size_t nitems, + gzFile file) { z_size_t len; gz_statep state; @@ -310,10 +284,7 @@ z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) } /* -- see zlib.h -- */ -int ZEXPORT gzputc(file, c) - gzFile file; - int c; -{ +int ZEXPORT gzputc(gzFile file, int c) { unsigned have; unsigned char buf[1]; gz_statep state; @@ -358,10 +329,7 @@ int ZEXPORT gzputc(file, c) } /* -- see zlib.h -- */ -int ZEXPORT gzputs(file, s) - gzFile file; - const char *s; -{ +int ZEXPORT gzputs(gzFile file, const char *s) { z_size_t len, put; gz_statep state; @@ -388,8 +356,7 @@ int ZEXPORT gzputs(file, s) #include /* -- see zlib.h -- */ -int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) -{ +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { int len; unsigned left; char *next; @@ -460,8 +427,7 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) return len; } -int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) -{ +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) { va_list va; int ret; @@ -474,13 +440,10 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) #else /* !STDC && !Z_HAVE_STDARG_H */ /* -- see zlib.h -- */ -int ZEXPORTVA gzprintf(file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) - gzFile file; - const char *format; - int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, - a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; -{ +int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, + int a4, int a5, int a6, int a7, int a8, int a9, int a10, + int a11, int a12, int a13, int a14, int a15, int a16, + int a17, int a18, int a19, int a20) { unsigned len, left; char *next; gz_statep state; @@ -562,10 +525,7 @@ int ZEXPORTVA gzprintf(file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, #endif /* -- see zlib.h -- */ -int ZEXPORT gzflush(file, flush) - gzFile file; - int flush; -{ +int ZEXPORT gzflush(gzFile file, int flush) { gz_statep state; /* get internal structure */ @@ -594,11 +554,7 @@ int ZEXPORT gzflush(file, flush) } /* -- see zlib.h -- */ -int ZEXPORT gzsetparams(file, level, strategy) - gzFile file; - int level; - int strategy; -{ +int ZEXPORT gzsetparams(gzFile file, int level, int strategy) { gz_statep state; z_streamp strm; @@ -609,7 +565,7 @@ int ZEXPORT gzsetparams(file, level, strategy) strm = &(state->strm); /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK) + if (state->mode != GZ_WRITE || state->err != Z_OK || state->direct) return Z_STREAM_ERROR; /* if no change is requested, then do nothing */ @@ -636,9 +592,7 @@ int ZEXPORT gzsetparams(file, level, strategy) } /* -- see zlib.h -- */ -int ZEXPORT gzclose_w(file) - gzFile file; -{ +int ZEXPORT gzclose_w(gzFile file) { int ret = Z_OK; gz_statep state; diff --git a/src/native/external/zlib/infback.c b/src/native/external/zlib/infback.c index babeaf1806f..e7b25b307a3 100644 --- a/src/native/external/zlib/infback.c +++ b/src/native/external/zlib/infback.c @@ -15,9 +15,6 @@ #include "inflate.h" #include "inffast.h" -/* function prototypes */ -local void fixedtables OF((struct inflate_state FAR *state)); - /* strm provides memory allocation functions in zalloc and zfree, or Z_NULL to use the library memory allocation functions. @@ -25,13 +22,9 @@ local void fixedtables OF((struct inflate_state FAR *state)); windowBits is in the range 8..15, and window is a user-supplied window and output buffer that is 2**windowBits bytes. */ -int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size) -z_streamp strm; -int windowBits; -unsigned char FAR *window; -const char *version; -int stream_size; -{ +int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, + unsigned char FAR *window, const char *version, + int stream_size) { struct inflate_state FAR *state; if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || @@ -80,9 +73,7 @@ int stream_size; used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ -local void fixedtables(state) -struct inflate_state FAR *state; -{ +local void fixedtables(struct inflate_state FAR *state) { #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; @@ -248,13 +239,8 @@ struct inflate_state FAR *state; inflateBack() can also return Z_STREAM_ERROR if the input parameters are not correct, i.e. strm is Z_NULL or the state was not initialized. */ -int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc) -z_streamp strm; -in_func in; -void FAR *in_desc; -out_func out; -void FAR *out_desc; -{ +int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc) { struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ @@ -632,9 +618,7 @@ void FAR *out_desc; return ret; } -int ZEXPORT inflateBackEnd(strm) -z_streamp strm; -{ +int ZEXPORT inflateBackEnd(z_streamp strm) { if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) return Z_STREAM_ERROR; ZFREE(strm, strm->state); diff --git a/src/native/external/zlib/inffast.c b/src/native/external/zlib/inffast.c index 1fec7f363fa..9354676e786 100644 --- a/src/native/external/zlib/inffast.c +++ b/src/native/external/zlib/inffast.c @@ -47,10 +47,7 @@ requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ -void ZLIB_INTERNAL inflate_fast(strm, start) -z_streamp strm; -unsigned start; /* inflate()'s starting value for strm->avail_out */ -{ +void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { struct inflate_state FAR *state; z_const unsigned char FAR *in; /* local strm->next_in */ z_const unsigned char FAR *last; /* have enough input while in < last */ diff --git a/src/native/external/zlib/inffast.h b/src/native/external/zlib/inffast.h index e5c1aa4ca8c..49c6d156c5c 100644 --- a/src/native/external/zlib/inffast.h +++ b/src/native/external/zlib/inffast.h @@ -8,4 +8,4 @@ subject to change. Applications should only use zlib.h. */ -void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); +void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start); diff --git a/src/native/external/zlib/inflate.c b/src/native/external/zlib/inflate.c index 8acbef44e99..94ecff015a9 100644 --- a/src/native/external/zlib/inflate.c +++ b/src/native/external/zlib/inflate.c @@ -91,20 +91,7 @@ # endif #endif -/* function prototypes */ -local int inflateStateCheck OF((z_streamp strm)); -local void fixedtables OF((struct inflate_state FAR *state)); -local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, - unsigned copy)); -#ifdef BUILDFIXED - void makefixed OF((void)); -#endif -local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, - unsigned len)); - -local int inflateStateCheck(strm) -z_streamp strm; -{ +local int inflateStateCheck(z_streamp strm) { struct inflate_state FAR *state; if (strm == Z_NULL || strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) @@ -116,9 +103,7 @@ z_streamp strm; return 0; } -int ZEXPORT inflateResetKeep(strm) -z_streamp strm; -{ +int ZEXPORT inflateResetKeep(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -142,9 +127,7 @@ z_streamp strm; return Z_OK; } -int ZEXPORT inflateReset(strm) -z_streamp strm; -{ +int ZEXPORT inflateReset(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -155,10 +138,7 @@ z_streamp strm; return inflateResetKeep(strm); } -int ZEXPORT inflateReset2(strm, windowBits) -z_streamp strm; -int windowBits; -{ +int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { int wrap; struct inflate_state FAR *state; @@ -195,12 +175,8 @@ int windowBits; return inflateReset(strm); } -int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) -z_streamp strm; -int windowBits; -const char *version; -int stream_size; -{ +int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, + const char *version, int stream_size) { int ret; struct inflate_state FAR *state; @@ -239,22 +215,17 @@ int stream_size; return ret; } -int ZEXPORT inflateInit_(strm, version, stream_size) -z_streamp strm; -const char *version; -int stream_size; -{ +int ZEXPORT inflateInit_(z_streamp strm, const char *version, + int stream_size) { return inflateInit2_(strm, DEF_WBITS, version, stream_size); } -int ZEXPORT inflatePrime(strm, bits, value) -z_streamp strm; -int bits; -int value; -{ +int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + if (bits == 0) + return Z_OK; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; @@ -278,9 +249,7 @@ int value; used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ -local void fixedtables(state) -struct inflate_state FAR *state; -{ +local void fixedtables(struct inflate_state FAR *state) { #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; @@ -342,7 +311,7 @@ struct inflate_state FAR *state; a.out > inffixed.h */ -void makefixed() +void makefixed(void) { unsigned low, size; struct inflate_state state; @@ -396,11 +365,7 @@ void makefixed() output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(strm, end, copy) -z_streamp strm; -const Bytef *end; -unsigned copy; -{ +local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { struct inflate_state FAR *state; unsigned dist; @@ -622,10 +587,7 @@ unsigned copy; will return Z_BUF_ERROR if it has not reached the end of the stream. */ -int ZEXPORT inflate(strm, flush) -z_streamp strm; -int flush; -{ +int ZEXPORT inflate(z_streamp strm, int flush) { struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ @@ -1301,9 +1263,7 @@ int flush; return ret; } -int ZEXPORT inflateEnd(strm) -z_streamp strm; -{ +int ZEXPORT inflateEnd(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1315,11 +1275,8 @@ z_streamp strm; return Z_OK; } -int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) -z_streamp strm; -Bytef *dictionary; -uInt *dictLength; -{ +int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, + uInt *dictLength) { struct inflate_state FAR *state; /* check state */ @@ -1338,11 +1295,8 @@ uInt *dictLength; return Z_OK; } -int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) -z_streamp strm; -const Bytef *dictionary; -uInt dictLength; -{ +int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, + uInt dictLength) { struct inflate_state FAR *state; unsigned long dictid; int ret; @@ -1373,10 +1327,7 @@ uInt dictLength; return Z_OK; } -int ZEXPORT inflateGetHeader(strm, head) -z_streamp strm; -gz_headerp head; -{ +int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) { struct inflate_state FAR *state; /* check state */ @@ -1401,11 +1352,8 @@ gz_headerp head; called again with more data and the *have state. *have is initialized to zero for the first call. */ -local unsigned syncsearch(have, buf, len) -unsigned FAR *have; -const unsigned char FAR *buf; -unsigned len; -{ +local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf, + unsigned len) { unsigned got; unsigned next; @@ -1424,9 +1372,7 @@ unsigned len; return next; } -int ZEXPORT inflateSync(strm) -z_streamp strm; -{ +int ZEXPORT inflateSync(z_streamp strm) { unsigned len; /* number of bytes to look at or looked at */ int flags; /* temporary to save header status */ unsigned long in, out; /* temporary to save total_in and total_out */ @@ -1441,7 +1387,7 @@ z_streamp strm; /* if first time, start search in bit buffer */ if (state->mode != SYNC) { state->mode = SYNC; - state->hold <<= state->bits & 7; + state->hold >>= state->bits & 7; state->bits -= state->bits & 7; len = 0; while (state->bits >= 8) { @@ -1482,9 +1428,7 @@ z_streamp strm; block. When decompressing, PPP checks that at the end of input packet, inflate is waiting for these length bytes. */ -int ZEXPORT inflateSyncPoint(strm) -z_streamp strm; -{ +int ZEXPORT inflateSyncPoint(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1492,10 +1436,7 @@ z_streamp strm; return state->mode == STORED && state->bits == 0; } -int ZEXPORT inflateCopy(dest, source) -z_streamp dest; -z_streamp source; -{ +int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { struct inflate_state FAR *state; struct inflate_state FAR *copy; unsigned char FAR *window; @@ -1539,10 +1480,7 @@ z_streamp source; return Z_OK; } -int ZEXPORT inflateUndermine(strm, subvert) -z_streamp strm; -int subvert; -{ +int ZEXPORT inflateUndermine(z_streamp strm, int subvert) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1557,10 +1495,7 @@ int subvert; #endif } -int ZEXPORT inflateValidate(strm, check) -z_streamp strm; -int check; -{ +int ZEXPORT inflateValidate(z_streamp strm, int check) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1572,9 +1507,7 @@ int check; return Z_OK; } -long ZEXPORT inflateMark(strm) -z_streamp strm; -{ +long ZEXPORT inflateMark(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) @@ -1585,9 +1518,7 @@ z_streamp strm; (state->mode == MATCH ? state->was - state->length : 0)); } -unsigned long ZEXPORT inflateCodesUsed(strm) -z_streamp strm; -{ +unsigned long ZEXPORT inflateCodesUsed(z_streamp strm) { struct inflate_state FAR *state; if (inflateStateCheck(strm)) return (unsigned long)-1; state = (struct inflate_state FAR *)strm->state; diff --git a/src/native/external/zlib/inftrees.c b/src/native/external/zlib/inftrees.c index 57d2793bec9..98cfe164458 100644 --- a/src/native/external/zlib/inftrees.c +++ b/src/native/external/zlib/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2022 Mark Adler + * Copyright (C) 1995-2024 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,7 +9,7 @@ #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.2.13 Copyright 1995-2022 Mark Adler "; + " inflate 1.3.1 Copyright 1995-2024 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -29,14 +29,9 @@ const char inflate_copyright[] = table index bits. It will differ if the request is greater than the longest code or if it is less than the shortest code. */ -int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) -codetype type; -unsigned short FAR *lens; -unsigned codes; -code FAR * FAR *table; -unsigned FAR *bits; -unsigned short FAR *work; -{ +int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work) { unsigned len; /* a code's length in bits */ unsigned sym; /* index of code symbols */ unsigned min, max; /* minimum and maximum code lengths */ @@ -62,7 +57,7 @@ unsigned short FAR *work; 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 194, 65}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 203, 77}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, diff --git a/src/native/external/zlib/inftrees.h b/src/native/external/zlib/inftrees.h index f53665311c1..396f74b5da7 100644 --- a/src/native/external/zlib/inftrees.h +++ b/src/native/external/zlib/inftrees.h @@ -41,8 +41,8 @@ typedef struct { examples/enough.c found in the zlib distribution. The arguments to that program are the number of symbols, the initial root table size, and the maximum bit length of a code. "enough 286 9 15" for literal/length codes - returns returns 852, and "enough 30 6 15" for distance codes returns 592. - The initial root table size (9 or 6) is found in the fifth argument of the + returns 852, and "enough 30 6 15" for distance codes returns 592. The + initial root table size (9 or 6) is found in the fifth argument of the inflate_table() calls in inflate.c and infback.c. If the root table size is changed, then these maximum sizes would be need to be recalculated and updated. */ @@ -57,6 +57,6 @@ typedef enum { DISTS } codetype; -int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, - unsigned codes, code FAR * FAR *table, - unsigned FAR *bits, unsigned short FAR *work)); +int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work); diff --git a/src/native/external/zlib/treebuild.xml b/src/native/external/zlib/treebuild.xml index 0017a45d3c5..930b00be4a8 100644 --- a/src/native/external/zlib/treebuild.xml +++ b/src/native/external/zlib/treebuild.xml @@ -1,6 +1,6 @@ - - + + zip compression library diff --git a/src/native/external/zlib/trees.c b/src/native/external/zlib/trees.c index 8a3eec559e5..979ae4100a0 100644 --- a/src/native/external/zlib/trees.c +++ b/src/native/external/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2021 Jean-loup Gailly + * Copyright (C) 1995-2024 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -122,39 +122,116 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -local const static_tree_desc static_l_desc = +#ifdef NO_INIT_GLOBAL_POINTERS +# define TCONST +#else +# define TCONST const +#endif + +local TCONST static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local const static_tree_desc static_d_desc = +local TCONST static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local const static_tree_desc static_bl_desc = +local TCONST static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== - * Local (static) routines in this file. + * Output a short LSB first on the stream. + * IN assertion: there is enough room in pendingBuf. + */ +#define put_short(s, w) { \ + put_byte(s, (uch)((w) & 0xff)); \ + put_byte(s, (uch)((ush)(w) >> 8)); \ +} + +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 */ +local unsigned bi_reverse(unsigned code, int len) { + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} -local void tr_static_init OF((void)); -local void init_block OF((deflate_state *s)); -local void pqdownheap OF((deflate_state *s, ct_data *tree, int k)); -local void gen_bitlen OF((deflate_state *s, tree_desc *desc)); -local void gen_codes OF((ct_data *tree, int max_code, ushf *bl_count)); -local void build_tree OF((deflate_state *s, tree_desc *desc)); -local void scan_tree OF((deflate_state *s, ct_data *tree, int max_code)); -local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); -local int build_bl_tree OF((deflate_state *s)); -local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, - int blcodes)); -local void compress_block OF((deflate_state *s, const ct_data *ltree, - const ct_data *dtree)); -local int detect_data_type OF((deflate_state *s)); -local unsigned bi_reverse OF((unsigned code, int len)); -local void bi_windup OF((deflate_state *s)); -local void bi_flush OF((deflate_state *s)); +/* =========================================================================== + * Flush the bit buffer, keeping at most 7 bits in it. + */ +local void bi_flush(deflate_state *s) { + if (s->bi_valid == 16) { + put_short(s, s->bi_buf); + s->bi_buf = 0; + s->bi_valid = 0; + } else if (s->bi_valid >= 8) { + put_byte(s, (Byte)s->bi_buf); + s->bi_buf >>= 8; + s->bi_valid -= 8; + } +} + +/* =========================================================================== + * Flush the bit buffer and align the output on a byte boundary + */ +local void bi_windup(deflate_state *s) { + if (s->bi_valid > 8) { + put_short(s, s->bi_buf); + } else if (s->bi_valid > 0) { + put_byte(s, (Byte)s->bi_buf); + } + s->bi_buf = 0; + s->bi_valid = 0; +#ifdef ZLIB_DEBUG + s->bits_sent = (s->bits_sent + 7) & ~7; +#endif +} + +/* =========================================================================== + * Generate the codes for a given tree and bit counts (which need not be + * optimal). + * IN assertion: the array bl_count contains the bit length statistics for + * the given tree and the field len is set for all tree elements. + * OUT assertion: the field code is set for all tree elements of non + * zero code length. + */ +local void gen_codes(ct_data *tree, int max_code, ushf *bl_count) { + ush next_code[MAX_BITS+1]; /* next code value for each bit length */ + unsigned code = 0; /* running code value */ + int bits; /* bit index */ + int n; /* code index */ + + /* The distribution counts are first used to generate the code values + * without bit reversal. + */ + for (bits = 1; bits <= MAX_BITS; bits++) { + code = (code + bl_count[bits - 1]) << 1; + next_code[bits] = (ush)code; + } + /* Check that the bit counts in bl_count are consistent. The last code + * must be all ones. + */ + Assert (code + bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1, + "inconsistent bit counts"); + Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); + + for (n = 0; n <= max_code; n++) { + int len = tree[n].Len; + if (len == 0) continue; + /* Now reverse the bits */ + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); + + Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", + n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len] - 1)); + } +} #ifdef GEN_TREES_H -local void gen_trees_header OF((void)); +local void gen_trees_header(void); #endif #ifndef ZLIB_DEBUG @@ -167,27 +244,12 @@ local void gen_trees_header OF((void)); send_bits(s, tree[c].Code, tree[c].Len); } #endif -/* =========================================================================== - * Output a short LSB first on the stream. - * IN assertion: there is enough room in pendingBuf. - */ -#define put_short(s, w) { \ - put_byte(s, (uch)((w) & 0xff)); \ - put_byte(s, (uch)((ush)(w) >> 8)); \ -} - /* =========================================================================== * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ #ifdef ZLIB_DEBUG -local void send_bits OF((deflate_state *s, int value, int length)); - -local void send_bits(s, value, length) - deflate_state *s; - int value; /* value to send */ - int length; /* number of bits */ -{ +local void send_bits(deflate_state *s, int value, int length) { Tracevv((stderr," l %2d v %4x ", length, value)); Assert(length > 0 && length <= 15, "invalid length"); s->bits_sent += (ulg)length; @@ -229,8 +291,7 @@ local void send_bits(s, value, length) /* =========================================================================== * Initialize the various 'constant' tables. */ -local void tr_static_init() -{ +local void tr_static_init(void) { #if defined(GEN_TREES_H) || !defined(STDC) static int static_init_done = 0; int n; /* iterates over tree elements */ @@ -323,8 +384,7 @@ local void tr_static_init() ((i) == (last)? "\n};\n\n" : \ ((i) % (width) == (width) - 1 ? ",\n" : ", ")) -void gen_trees_header() -{ +void gen_trees_header(void) { FILE *header = fopen("trees.h", "w"); int i; @@ -373,12 +433,26 @@ void gen_trees_header() } #endif /* GEN_TREES_H */ +/* =========================================================================== + * Initialize a new block. + */ +local void init_block(deflate_state *s) { + int n; /* iterates over tree elements */ + + /* Initialize the trees. */ + for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; + for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; + for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; + + s->dyn_ltree[END_BLOCK].Freq = 1; + s->opt_len = s->static_len = 0L; + s->sym_next = s->matches = 0; +} + /* =========================================================================== * Initialize the tree data structures for a new zlib stream. */ -void ZLIB_INTERNAL _tr_init(s) - deflate_state *s; -{ +void ZLIB_INTERNAL _tr_init(deflate_state *s) { tr_static_init(); s->l_desc.dyn_tree = s->dyn_ltree; @@ -401,24 +475,6 @@ void ZLIB_INTERNAL _tr_init(s) init_block(s); } -/* =========================================================================== - * Initialize a new block. - */ -local void init_block(s) - deflate_state *s; -{ - int n; /* iterates over tree elements */ - - /* Initialize the trees. */ - for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; - for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; - for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; - - s->dyn_ltree[END_BLOCK].Freq = 1; - s->opt_len = s->static_len = 0L; - s->sym_next = s->matches = 0; -} - #define SMALLEST 1 /* Index within the heap array of least frequent node in the Huffman tree */ @@ -448,11 +504,7 @@ local void init_block(s) * when the heap property is re-established (each father smaller than its * two sons). */ -local void pqdownheap(s, tree, k) - deflate_state *s; - ct_data *tree; /* the tree to restore */ - int k; /* node to move down */ -{ +local void pqdownheap(deflate_state *s, ct_data *tree, int k) { int v = s->heap[k]; int j = k << 1; /* left son of k */ while (j <= s->heap_len) { @@ -483,10 +535,7 @@ local void pqdownheap(s, tree, k) * The length opt_len is updated; static_len is also updated if stree is * not null. */ -local void gen_bitlen(s, desc) - deflate_state *s; - tree_desc *desc; /* the tree descriptor */ -{ +local void gen_bitlen(deflate_state *s, tree_desc *desc) { ct_data *tree = desc->dyn_tree; int max_code = desc->max_code; const ct_data *stree = desc->stat_desc->static_tree; @@ -561,48 +610,9 @@ local void gen_bitlen(s, desc) } } -/* =========================================================================== - * Generate the codes for a given tree and bit counts (which need not be - * optimal). - * IN assertion: the array bl_count contains the bit length statistics for - * the given tree and the field len is set for all tree elements. - * OUT assertion: the field code is set for all tree elements of non - * zero code length. - */ -local void gen_codes(tree, max_code, bl_count) - ct_data *tree; /* the tree to decorate */ - int max_code; /* largest code with non zero frequency */ - ushf *bl_count; /* number of codes at each bit length */ -{ - ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - unsigned code = 0; /* running code value */ - int bits; /* bit index */ - int n; /* code index */ - - /* The distribution counts are first used to generate the code values - * without bit reversal. - */ - for (bits = 1; bits <= MAX_BITS; bits++) { - code = (code + bl_count[bits - 1]) << 1; - next_code[bits] = (ush)code; - } - /* Check that the bit counts in bl_count are consistent. The last code - * must be all ones. - */ - Assert (code + bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1, - "inconsistent bit counts"); - Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); - - for (n = 0; n <= max_code; n++) { - int len = tree[n].Len; - if (len == 0) continue; - /* Now reverse the bits */ - tree[n].Code = (ush)bi_reverse(next_code[len]++, len); - - Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", - n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len] - 1)); - } -} +#ifdef DUMP_BL_TREE +# include +#endif /* =========================================================================== * Construct one Huffman tree and assigns the code bit strings and lengths. @@ -612,10 +622,7 @@ local void gen_codes(tree, max_code, bl_count) * and corresponding code. The length opt_len is updated; static_len is * also updated if stree is not null. The field max_code is set. */ -local void build_tree(s, desc) - deflate_state *s; - tree_desc *desc; /* the tree descriptor */ -{ +local void build_tree(deflate_state *s, tree_desc *desc) { ct_data *tree = desc->dyn_tree; const ct_data *stree = desc->stat_desc->static_tree; int elems = desc->stat_desc->elems; @@ -700,11 +707,7 @@ local void build_tree(s, desc) * Scan a literal or distance tree to determine the frequencies of the codes * in the bit length tree. */ -local void scan_tree(s, tree, max_code) - deflate_state *s; - ct_data *tree; /* the tree to be scanned */ - int max_code; /* and its largest code of non zero frequency */ -{ +local void scan_tree(deflate_state *s, ct_data *tree, int max_code) { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ @@ -745,11 +748,7 @@ local void scan_tree(s, tree, max_code) * Send a literal or distance tree in compressed form, using the codes in * bl_tree. */ -local void send_tree(s, tree, max_code) - deflate_state *s; - ct_data *tree; /* the tree to be scanned */ - int max_code; /* and its largest code of non zero frequency */ -{ +local void send_tree(deflate_state *s, ct_data *tree, int max_code) { int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ @@ -796,9 +795,7 @@ local void send_tree(s, tree, max_code) * Construct the Huffman tree for the bit lengths and return the index in * bl_order of the last bit length code to send. */ -local int build_bl_tree(s) - deflate_state *s; -{ +local int build_bl_tree(deflate_state *s) { int max_blindex; /* index of last bit length code of non zero freq */ /* Determine the bit length frequencies for literal and distance trees */ @@ -831,10 +828,8 @@ local int build_bl_tree(s) * lengths of the bit length codes, the literal tree and the distance tree. * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. */ -local void send_all_trees(s, lcodes, dcodes, blcodes) - deflate_state *s; - int lcodes, dcodes, blcodes; /* number of codes for each tree */ -{ +local void send_all_trees(deflate_state *s, int lcodes, int dcodes, + int blcodes) { int rank; /* index in bl_order */ Assert (lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes"); @@ -860,12 +855,8 @@ local void send_all_trees(s, lcodes, dcodes, blcodes) /* =========================================================================== * Send a stored block */ -void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) - deflate_state *s; - charf *buf; /* input block */ - ulg stored_len; /* length of input block */ - int last; /* one if this is the last block for a file */ -{ +void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, + ulg stored_len, int last) { send_bits(s, (STORED_BLOCK<<1) + last, 3); /* send block type */ bi_windup(s); /* align on byte boundary */ put_short(s, (ush)stored_len); @@ -884,9 +875,7 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) /* =========================================================================== * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) */ -void ZLIB_INTERNAL _tr_flush_bits(s) - deflate_state *s; -{ +void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s) { bi_flush(s); } @@ -894,9 +883,7 @@ void ZLIB_INTERNAL _tr_flush_bits(s) * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. */ -void ZLIB_INTERNAL _tr_align(s) - deflate_state *s; -{ +void ZLIB_INTERNAL _tr_align(deflate_state *s) { send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); #ifdef ZLIB_DEBUG @@ -905,16 +892,108 @@ void ZLIB_INTERNAL _tr_align(s) bi_flush(s); } +/* =========================================================================== + * Send the block data compressed using the given Huffman trees + */ +local void compress_block(deflate_state *s, const ct_data *ltree, + const ct_data *dtree) { + unsigned dist; /* distance of matched string */ + int lc; /* match length or unmatched char (if dist == 0) */ + unsigned sx = 0; /* running index in symbol buffers */ + unsigned code; /* the code to send */ + int extra; /* number of extra bits to send */ + + if (s->sym_next != 0) do { +#ifdef LIT_MEM + dist = s->d_buf[sx]; + lc = s->l_buf[sx++]; +#else + dist = s->sym_buf[sx++] & 0xff; + dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; + lc = s->sym_buf[sx++]; +#endif + if (dist == 0) { + send_code(s, lc, ltree); /* send a literal byte */ + Tracecv(isgraph(lc), (stderr," '%c' ", lc)); + } else { + /* Here, lc is the match length - MIN_MATCH */ + code = _length_code[lc]; + send_code(s, code + LITERALS + 1, ltree); /* send length code */ + extra = extra_lbits[code]; + if (extra != 0) { + lc -= base_length[code]; + send_bits(s, lc, extra); /* send the extra length bits */ + } + dist--; /* dist is now the match distance - 1 */ + code = d_code(dist); + Assert (code < D_CODES, "bad d_code"); + + send_code(s, code, dtree); /* send the distance code */ + extra = extra_dbits[code]; + if (extra != 0) { + dist -= (unsigned)base_dist[code]; + send_bits(s, dist, extra); /* send the extra distance bits */ + } + } /* literal or match pair ? */ + + /* Check for no overlay of pending_buf on needed symbols */ +#ifdef LIT_MEM + Assert(s->pending < 2 * (s->lit_bufsize + sx), "pendingBuf overflow"); +#else + Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); +#endif + + } while (sx < s->sym_next); + + send_code(s, END_BLOCK, ltree); +} + +/* =========================================================================== + * Check if the data type is TEXT or BINARY, using the following algorithm: + * - TEXT if the two conditions below are satisfied: + * a) There are no non-portable control characters belonging to the + * "block list" (0..6, 14..25, 28..31). + * b) There is at least one printable character belonging to the + * "allow list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). + * - BINARY otherwise. + * - The following partially-portable control characters form a + * "gray list" that is ignored in this detection algorithm: + * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). + * IN assertion: the fields Freq of dyn_ltree are set. + */ +local int detect_data_type(deflate_state *s) { + /* block_mask is the bit mask of block-listed bytes + * set bits 0..6, 14..25, and 28..31 + * 0xf3ffc07f = binary 11110011111111111100000001111111 + */ + unsigned long block_mask = 0xf3ffc07fUL; + int n; + + /* Check for non-textual ("block-listed") bytes. */ + for (n = 0; n <= 31; n++, block_mask >>= 1) + if ((block_mask & 1) && (s->dyn_ltree[n].Freq != 0)) + return Z_BINARY; + + /* Check for textual ("allow-listed") bytes. */ + if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 + || s->dyn_ltree[13].Freq != 0) + return Z_TEXT; + for (n = 32; n < LITERALS; n++) + if (s->dyn_ltree[n].Freq != 0) + return Z_TEXT; + + /* There are no "block-listed" or "allow-listed" bytes: + * this stream either is empty or has tolerated ("gray-listed") bytes only. + */ + return Z_BINARY; +} + /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static * trees or store, and write out the encoded block. */ -void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) - deflate_state *s; - charf *buf; /* input block, or NULL if too old */ - ulg stored_len; /* length of input block */ - int last; /* one if this is the last block for a file */ -{ +void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, + ulg stored_len, int last) { ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ int max_blindex = 0; /* index of last bit length code of non zero freq */ @@ -1011,14 +1090,15 @@ void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) * Save the match info and tally the frequency counts. Return true if * the current block must be flushed. */ -int ZLIB_INTERNAL _tr_tally(s, dist, lc) - deflate_state *s; - unsigned dist; /* distance of matched string */ - unsigned lc; /* match length - MIN_MATCH or unmatched char (dist==0) */ -{ +int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) { +#ifdef LIT_MEM + s->d_buf[s->sym_next] = (ush)dist; + s->l_buf[s->sym_next++] = (uch)lc; +#else s->sym_buf[s->sym_next++] = (uch)dist; s->sym_buf[s->sym_next++] = (uch)(dist >> 8); s->sym_buf[s->sym_next++] = (uch)lc; +#endif if (dist == 0) { /* lc is the unmatched char */ s->dyn_ltree[lc].Freq++; @@ -1035,147 +1115,3 @@ int ZLIB_INTERNAL _tr_tally(s, dist, lc) } return (s->sym_next == s->sym_end); } - -/* =========================================================================== - * Send the block data compressed using the given Huffman trees - */ -local void compress_block(s, ltree, dtree) - deflate_state *s; - const ct_data *ltree; /* literal tree */ - const ct_data *dtree; /* distance tree */ -{ - unsigned dist; /* distance of matched string */ - int lc; /* match length or unmatched char (if dist == 0) */ - unsigned sx = 0; /* running index in sym_buf */ - unsigned code; /* the code to send */ - int extra; /* number of extra bits to send */ - - if (s->sym_next != 0) do { - dist = s->sym_buf[sx++] & 0xff; - dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; - lc = s->sym_buf[sx++]; - if (dist == 0) { - send_code(s, lc, ltree); /* send a literal byte */ - Tracecv(isgraph(lc), (stderr," '%c' ", lc)); - } else { - /* Here, lc is the match length - MIN_MATCH */ - code = _length_code[lc]; - send_code(s, code + LITERALS + 1, ltree); /* send length code */ - extra = extra_lbits[code]; - if (extra != 0) { - lc -= base_length[code]; - send_bits(s, lc, extra); /* send the extra length bits */ - } - dist--; /* dist is now the match distance - 1 */ - code = d_code(dist); - Assert (code < D_CODES, "bad d_code"); - - send_code(s, code, dtree); /* send the distance code */ - extra = extra_dbits[code]; - if (extra != 0) { - dist -= (unsigned)base_dist[code]; - send_bits(s, dist, extra); /* send the extra distance bits */ - } - } /* literal or match pair ? */ - - /* Check that the overlay between pending_buf and sym_buf is ok: */ - Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); - - } while (sx < s->sym_next); - - send_code(s, END_BLOCK, ltree); -} - -/* =========================================================================== - * Check if the data type is TEXT or BINARY, using the following algorithm: - * - TEXT if the two conditions below are satisfied: - * a) There are no non-portable control characters belonging to the - * "block list" (0..6, 14..25, 28..31). - * b) There is at least one printable character belonging to the - * "allow list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). - * - BINARY otherwise. - * - The following partially-portable control characters form a - * "gray list" that is ignored in this detection algorithm: - * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). - * IN assertion: the fields Freq of dyn_ltree are set. - */ -local int detect_data_type(s) - deflate_state *s; -{ - /* block_mask is the bit mask of block-listed bytes - * set bits 0..6, 14..25, and 28..31 - * 0xf3ffc07f = binary 11110011111111111100000001111111 - */ - unsigned long block_mask = 0xf3ffc07fUL; - int n; - - /* Check for non-textual ("block-listed") bytes. */ - for (n = 0; n <= 31; n++, block_mask >>= 1) - if ((block_mask & 1) && (s->dyn_ltree[n].Freq != 0)) - return Z_BINARY; - - /* Check for textual ("allow-listed") bytes. */ - if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 - || s->dyn_ltree[13].Freq != 0) - return Z_TEXT; - for (n = 32; n < LITERALS; n++) - if (s->dyn_ltree[n].Freq != 0) - return Z_TEXT; - - /* There are no "block-listed" or "allow-listed" bytes: - * this stream either is empty or has tolerated ("gray-listed") bytes only. - */ - return Z_BINARY; -} - -/* =========================================================================== - * Reverse the first len bits of a code, using straightforward code (a faster - * method would use a table) - * IN assertion: 1 <= len <= 15 - */ -local unsigned bi_reverse(code, len) - unsigned code; /* the value to invert */ - int len; /* its bit length */ -{ - register unsigned res = 0; - do { - res |= code & 1; - code >>= 1, res <<= 1; - } while (--len > 0); - return res >> 1; -} - -/* =========================================================================== - * Flush the bit buffer, keeping at most 7 bits in it. - */ -local void bi_flush(s) - deflate_state *s; -{ - if (s->bi_valid == 16) { - put_short(s, s->bi_buf); - s->bi_buf = 0; - s->bi_valid = 0; - } else if (s->bi_valid >= 8) { - put_byte(s, (Byte)s->bi_buf); - s->bi_buf >>= 8; - s->bi_valid -= 8; - } -} - -/* =========================================================================== - * Flush the bit buffer and align the output on a byte boundary - */ -local void bi_windup(s) - deflate_state *s; -{ - if (s->bi_valid > 8) { - put_short(s, s->bi_buf); - } else if (s->bi_valid > 0) { - put_byte(s, (Byte)s->bi_buf); - } - s->bi_buf = 0; - s->bi_valid = 0; -#ifdef ZLIB_DEBUG - s->bits_sent = (s->bits_sent + 7) & ~7; -#endif -} diff --git a/src/native/external/zlib/uncompr.c b/src/native/external/zlib/uncompr.c index f9532f46c1a..5e256663b45 100644 --- a/src/native/external/zlib/uncompr.c +++ b/src/native/external/zlib/uncompr.c @@ -24,12 +24,8 @@ Z_DATA_ERROR if the input data was corrupted, including if the input data is an incomplete zlib stream. */ -int ZEXPORT uncompress2(dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong *sourceLen; -{ +int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong *sourceLen) { z_stream stream; int err; const uInt max = (uInt)-1; @@ -83,11 +79,7 @@ int ZEXPORT uncompress2(dest, destLen, source, sourceLen) err; } -int ZEXPORT uncompress(dest, destLen, source, sourceLen) - Bytef *dest; - uLongf *destLen; - const Bytef *source; - uLong sourceLen; -{ +int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, + uLong sourceLen) { return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/src/native/external/zlib/zconf.h b/src/native/external/zlib/zconf.h index bf977d3e70a..62adc8d8431 100644 --- a/src/native/external/zlib/zconf.h +++ b/src/native/external/zlib/zconf.h @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -241,7 +241,11 @@ #endif #ifdef Z_SOLO - typedef unsigned long z_size_t; +# ifdef _WIN64 + typedef unsigned long long z_size_t; +# else + typedef unsigned long z_size_t; +# endif #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -296,14 +300,6 @@ # endif #endif -#ifndef Z_ARG /* function prototypes for stdarg */ -# if defined(STDC) || defined(Z_HAVE_STDARG_H) -# define Z_ARG(args) args -# else -# define Z_ARG(args) () -# endif -#endif - /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -520,7 +516,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# if defined(_WIN32) && !defined(__GNUC__) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zconf.h.cmakein b/src/native/external/zlib/zconf.h.cmakein index 247ba2461dd..0abe3bc9d8f 100644 --- a/src/native/external/zlib/zconf.h.cmakein +++ b/src/native/external/zlib/zconf.h.cmakein @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -243,7 +243,11 @@ #endif #ifdef Z_SOLO - typedef unsigned long z_size_t; +# ifdef _WIN64 + typedef unsigned long long z_size_t; +# else + typedef unsigned long z_size_t; +# endif #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -298,14 +302,6 @@ # endif #endif -#ifndef Z_ARG /* function prototypes for stdarg */ -# if defined(STDC) || defined(Z_HAVE_STDARG_H) -# define Z_ARG(args) args -# else -# define Z_ARG(args) () -# endif -#endif - /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -522,7 +518,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# if defined(_WIN32) && !defined(__GNUC__) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zconf.h.in b/src/native/external/zlib/zconf.h.in index bf977d3e70a..62adc8d8431 100644 --- a/src/native/external/zlib/zconf.h.in +++ b/src/native/external/zlib/zconf.h.in @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -241,7 +241,11 @@ #endif #ifdef Z_SOLO - typedef unsigned long z_size_t; +# ifdef _WIN64 + typedef unsigned long long z_size_t; +# else + typedef unsigned long z_size_t; +# endif #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -296,14 +300,6 @@ # endif #endif -#ifndef Z_ARG /* function prototypes for stdarg */ -# if defined(STDC) || defined(Z_HAVE_STDARG_H) -# define Z_ARG(args) args -# else -# define Z_ARG(args) () -# endif -#endif - /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -520,7 +516,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# if defined(_WIN32) && !defined(__GNUC__) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zlib.3 b/src/native/external/zlib/zlib.3 index 6f6e91404df..c716020ea9c 100644 --- a/src/native/external/zlib/zlib.3 +++ b/src/native/external/zlib/zlib.3 @@ -1,4 +1,4 @@ -.TH ZLIB 3 "13 Oct 2022" +.TH ZLIB 3 "22 Jan 2024" .SH NAME zlib \- compression/decompression library .SH SYNOPSIS @@ -105,9 +105,9 @@ before asking for help. Send questions and/or comments to zlib@gzip.org, or (for the Windows DLL version) to Gilles Vollant (info@winimage.com). .SH AUTHORS AND LICENSE -Version 1.2.13 +Version 1.3.1 .LP -Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler +Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler .LP This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/src/native/external/zlib/zlib.h b/src/native/external/zlib/zlib.h index 953cb5012dc..8d4b932eaf6 100644 --- a/src/native/external/zlib/zlib.h +++ b/src/native/external/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.2.13, October 13th, 2022 + version 1.3.1, January 22nd, 2024 - Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -37,11 +37,11 @@ extern "C" { #endif -#define ZLIB_VERSION "1.2.13" -#define ZLIB_VERNUM 0x12d0 +#define ZLIB_VERSION "1.3.1" +#define ZLIB_VERNUM 0x1310 #define ZLIB_VER_MAJOR 1 -#define ZLIB_VER_MINOR 2 -#define ZLIB_VER_REVISION 13 +#define ZLIB_VER_MINOR 3 +#define ZLIB_VER_REVISION 1 #define ZLIB_VER_SUBREVISION 0 /* @@ -78,8 +78,8 @@ extern "C" { even in the case of corrupted input. */ -typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); -typedef void (*free_func) OF((voidpf opaque, voidpf address)); +typedef voidpf (*alloc_func)(voidpf opaque, uInt items, uInt size); +typedef void (*free_func)(voidpf opaque, voidpf address); struct internal_state; @@ -217,7 +217,7 @@ typedef gz_header FAR *gz_headerp; /* basic functions */ -ZEXTERN const char * ZEXPORT zlibVersion OF((void)); +ZEXTERN const char * ZEXPORT zlibVersion(void); /* The application can compare zlibVersion and ZLIB_VERSION for consistency. If the first character differs, the library code actually used is not compatible with the zlib.h header file used by the application. This check @@ -225,12 +225,12 @@ ZEXTERN const char * ZEXPORT zlibVersion OF((void)); */ /* -ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); +ZEXTERN int ZEXPORT deflateInit(z_streamp strm, int level); Initializes the internal stream state for compression. The fields zalloc, zfree and opaque must be initialized before by the caller. If zalloc and zfree are set to Z_NULL, deflateInit updates them to use default - allocation functions. + allocation functions. total_in, total_out, adler, and msg are initialized. The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: 1 gives best speed, 9 gives best compression, 0 gives no compression at all @@ -247,7 +247,7 @@ ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); */ -ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); +ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); /* deflate compresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce @@ -320,8 +320,8 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); with the same value of the flush parameter and more output space (updated avail_out), until the flush is complete (deflate returns with non-zero avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that - avail_out is greater than six to avoid repeated flush markers due to - avail_out == 0 on return. + avail_out is greater than six when the flush marker begins, in order to avoid + repeated flush markers upon calling deflate() again when avail_out == 0. If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was @@ -360,7 +360,7 @@ ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); */ -ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); +ZEXTERN int ZEXPORT deflateEnd(z_streamp strm); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending @@ -375,7 +375,7 @@ ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); /* -ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); +ZEXTERN int ZEXPORT inflateInit(z_streamp strm); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by @@ -383,7 +383,8 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); read or consumed. The allocation of a sliding window will be deferred to the first call of inflate (if the decompression does not complete on the first call). If zalloc and zfree are set to Z_NULL, inflateInit updates - them to use default allocation functions. + them to use default allocation functions. total_in, total_out, adler, and + msg are initialized. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -397,7 +398,7 @@ ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); */ -ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); +ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush); /* inflate decompresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce @@ -517,7 +518,7 @@ ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); */ -ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); +ZEXTERN int ZEXPORT inflateEnd(z_streamp strm); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending @@ -535,12 +536,12 @@ ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); */ /* -ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, - int level, - int method, - int windowBits, - int memLevel, - int strategy)); +ZEXTERN int ZEXPORT deflateInit2(z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy); This is another version of deflateInit with more compression options. The fields zalloc, zfree and opaque must be initialized before by the caller. @@ -607,9 +608,9 @@ ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, compression: this will be done by deflate(). */ -ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, - const Bytef *dictionary, - uInt dictLength)); +ZEXTERN int ZEXPORT deflateSetDictionary(z_streamp strm, + const Bytef *dictionary, + uInt dictLength); /* Initializes the compression dictionary from the given byte sequence without producing any compressed output. When using the zlib format, this @@ -651,9 +652,9 @@ ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, not perform any compression: this will be done by deflate(). */ -ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, - Bytef *dictionary, - uInt *dictLength)); +ZEXTERN int ZEXPORT deflateGetDictionary(z_streamp strm, + Bytef *dictionary, + uInt *dictLength); /* Returns the sliding dictionary being maintained by deflate. dictLength is set to the number of bytes in the dictionary, and that many bytes are copied @@ -673,8 +674,8 @@ ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, stream state is inconsistent. */ -ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, - z_streamp source)); +ZEXTERN int ZEXPORT deflateCopy(z_streamp dest, + z_streamp source); /* Sets the destination stream as a complete copy of the source stream. @@ -691,20 +692,20 @@ ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, destination. */ -ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); +ZEXTERN int ZEXPORT deflateReset(z_streamp strm); /* This function is equivalent to deflateEnd followed by deflateInit, but does not free and reallocate the internal compression state. The stream will leave the compression level and any other attributes that may have been - set unchanged. + set unchanged. total_in, total_out, adler, and msg are initialized. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ -ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, - int level, - int strategy)); +ZEXTERN int ZEXPORT deflateParams(z_streamp strm, + int level, + int strategy); /* Dynamically update the compression level and compression strategy. The interpretation of level and strategy is as in deflateInit2(). This can be @@ -729,7 +730,7 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, Then no more input data should be provided before the deflateParams() call. If this is done, the old level and strategy will be applied to the data compressed before deflateParams(), and the new level and strategy will be - applied to the the data compressed after deflateParams(). + applied to the data compressed after deflateParams(). deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if @@ -740,11 +741,11 @@ ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, retried with more output space. */ -ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, - int good_length, - int max_lazy, - int nice_length, - int max_chain)); +ZEXTERN int ZEXPORT deflateTune(z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain); /* Fine tune deflate's internal compression parameters. This should only be used by someone who understands the algorithm used by zlib's deflate for @@ -757,8 +758,8 @@ ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. */ -ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, - uLong sourceLen)); +ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, + uLong sourceLen); /* deflateBound() returns an upper bound on the compressed size after deflation of sourceLen bytes. It must be called after deflateInit() or @@ -772,9 +773,9 @@ ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, than Z_FINISH or Z_NO_FLUSH are used. */ -ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, - unsigned *pending, - int *bits)); +ZEXTERN int ZEXPORT deflatePending(z_streamp strm, + unsigned *pending, + int *bits); /* deflatePending() returns the number of bytes and bits of output that have been generated, but not yet provided in the available output. The bytes not @@ -787,9 +788,9 @@ ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, stream state was inconsistent. */ -ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, - int bits, - int value)); +ZEXTERN int ZEXPORT deflatePrime(z_streamp strm, + int bits, + int value); /* deflatePrime() inserts bits in the deflate output stream. The intent is that this function is used to start off the deflate output with the bits @@ -804,8 +805,8 @@ ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, source stream state was inconsistent. */ -ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, - gz_headerp head)); +ZEXTERN int ZEXPORT deflateSetHeader(z_streamp strm, + gz_headerp head); /* deflateSetHeader() provides gzip header information for when a gzip stream is requested by deflateInit2(). deflateSetHeader() may be called @@ -821,16 +822,17 @@ ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, gzip file" and give up. If deflateSetHeader is not used, the default gzip header has text false, - the time set to zero, and os set to 255, with no extra, name, or comment - fields. The gzip header is returned to the default state by deflateReset(). + the time set to zero, and os set to the current operating system, with no + extra, name, or comment fields. The gzip header is returned to the default + state by deflateReset(). deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ /* -ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, - int windowBits)); +ZEXTERN int ZEXPORT inflateInit2(z_streamp strm, + int windowBits); This is another version of inflateInit with an extra parameter. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized @@ -883,9 +885,9 @@ ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, deferred until inflate() is called. */ -ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, - const Bytef *dictionary, - uInt dictLength)); +ZEXTERN int ZEXPORT inflateSetDictionary(z_streamp strm, + const Bytef *dictionary, + uInt dictLength); /* Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, @@ -906,9 +908,9 @@ ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, inflate(). */ -ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, - Bytef *dictionary, - uInt *dictLength)); +ZEXTERN int ZEXPORT inflateGetDictionary(z_streamp strm, + Bytef *dictionary, + uInt *dictLength); /* Returns the sliding dictionary being maintained by inflate. dictLength is set to the number of bytes in the dictionary, and that many bytes are copied @@ -921,7 +923,7 @@ ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, stream state is inconsistent. */ -ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); +ZEXTERN int ZEXPORT inflateSync(z_streamp strm); /* Skips invalid compressed data until a possible full flush point (see above for the description of deflate with Z_FULL_FLUSH) can be found, or until all @@ -934,14 +936,14 @@ ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); inflateSync returns Z_OK if a possible full flush point has been found, Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. - In the success case, the application may save the current current value of - total_in which indicates where valid compressed data was found. In the - error case, the application may repeatedly call inflateSync, providing more - input each time, until success or end of the input data. + In the success case, the application may save the current value of total_in + which indicates where valid compressed data was found. In the error case, + the application may repeatedly call inflateSync, providing more input each + time, until success or end of the input data. */ -ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, - z_streamp source)); +ZEXTERN int ZEXPORT inflateCopy(z_streamp dest, + z_streamp source); /* Sets the destination stream as a complete copy of the source stream. @@ -956,18 +958,19 @@ ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, destination. */ -ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); +ZEXTERN int ZEXPORT inflateReset(z_streamp strm); /* This function is equivalent to inflateEnd followed by inflateInit, but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. + total_in, total_out, adler, and msg are initialized. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ -ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, - int windowBits)); +ZEXTERN int ZEXPORT inflateReset2(z_streamp strm, + int windowBits); /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted @@ -980,9 +983,9 @@ ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, the windowBits parameter is invalid. */ -ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, - int bits, - int value)); +ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, + int bits, + int value); /* This function inserts bits in the inflate input stream. The intent is that this function is used to start inflating at a bit position in the @@ -1001,7 +1004,7 @@ ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, stream state was inconsistent. */ -ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +ZEXTERN long ZEXPORT inflateMark(z_streamp strm); /* This function returns two values, one in the lower 16 bits of the return value, and the other in the remaining upper bits, obtained by shifting the @@ -1029,8 +1032,8 @@ ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); source stream state was inconsistent. */ -ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, - gz_headerp head)); +ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm, + gz_headerp head); /* inflateGetHeader() requests that gzip header information be stored in the provided gz_header structure. inflateGetHeader() may be called after @@ -1070,8 +1073,8 @@ ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, */ /* -ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, - unsigned char FAR *window)); +ZEXTERN int ZEXPORT inflateBackInit(z_streamp strm, int windowBits, + unsigned char FAR *window); Initialize the internal stream state for decompression using inflateBack() calls. The fields zalloc, zfree and opaque in strm must be initialized @@ -1091,13 +1094,13 @@ ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, the version of the header file. */ -typedef unsigned (*in_func) OF((void FAR *, - z_const unsigned char FAR * FAR *)); -typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); +typedef unsigned (*in_func)(void FAR *, + z_const unsigned char FAR * FAR *); +typedef int (*out_func)(void FAR *, unsigned char FAR *, unsigned); -ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, - in_func in, void FAR *in_desc, - out_func out, void FAR *out_desc)); +ZEXTERN int ZEXPORT inflateBack(z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc); /* inflateBack() does a raw inflate with a single call using a call-back interface for input and output. This is potentially more efficient than @@ -1165,7 +1168,7 @@ ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, cannot return Z_OK. */ -ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); +ZEXTERN int ZEXPORT inflateBackEnd(z_streamp strm); /* All memory allocated by inflateBackInit() is freed. @@ -1173,7 +1176,7 @@ ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); state was inconsistent. */ -ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); +ZEXTERN uLong ZEXPORT zlibCompileFlags(void); /* Return flags indicating compile-time options. Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: @@ -1226,8 +1229,8 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); you need special options. */ -ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); +ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); /* Compresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size @@ -1241,9 +1244,9 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, buffer. */ -ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen, - int level)); +ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level); /* Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte @@ -1257,15 +1260,15 @@ ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, Z_STREAM_ERROR if the level parameter is invalid. */ -ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); +ZEXTERN uLong ZEXPORT compressBound(uLong sourceLen); /* compressBound() returns an upper bound on the compressed size after compress() or compress2() on sourceLen bytes. It would be used before a compress() or compress2() call to allocate the destination buffer. */ -ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); +ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen); /* Decompresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size @@ -1282,8 +1285,8 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ -ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong *sourceLen)); +ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen); /* Same as uncompress, except that sourceLen is a pointer, where the length of the source is *sourceLen. On return, *sourceLen is the number of @@ -1302,7 +1305,7 @@ ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ /* -ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); +ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); Open the gzip (.gz) file at path for reading and decompressing, or compressing and writing. The mode parameter is as in fopen ("rb" or "wb") @@ -1339,7 +1342,7 @@ ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); file could not be opened. */ -ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); +ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode); /* Associate a gzFile with the file descriptor fd. File descriptors are obtained from calls like open, dup, creat, pipe or fileno (if the file has @@ -1362,7 +1365,7 @@ ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); will not detect if fd is invalid (unless fd is -1). */ -ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); +ZEXTERN int ZEXPORT gzbuffer(gzFile file, unsigned size); /* Set the internal buffer size used by this library's functions for file to size. The default buffer size is 8192 bytes. This function must be called @@ -1378,7 +1381,7 @@ ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); too late. */ -ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); +ZEXTERN int ZEXPORT gzsetparams(gzFile file, int level, int strategy); /* Dynamically update the compression level and strategy for file. See the description of deflateInit2 for the meaning of these parameters. Previously @@ -1389,7 +1392,7 @@ ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); or Z_MEM_ERROR if there is a memory allocation error. */ -ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); +ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len); /* Read and decompress up to len uncompressed bytes from file into buf. If the input file is not in gzip format, gzread copies the given number of @@ -1419,8 +1422,8 @@ ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); Z_STREAM_ERROR. */ -ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, - gzFile file)); +ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, + gzFile file); /* Read and decompress up to nitems items of size size from file into buf, otherwise operating as gzread() does. This duplicates the interface of @@ -1445,14 +1448,14 @@ ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, file, resetting and retrying on end-of-file, when size is not 1. */ -ZEXTERN int ZEXPORT gzwrite OF((gzFile file, voidpc buf, unsigned len)); +ZEXTERN int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len); /* Compress and write the len uncompressed bytes at buf to file. gzwrite returns the number of uncompressed bytes written or 0 in case of error. */ -ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, - z_size_t nitems, gzFile file)); +ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, + z_size_t nitems, gzFile file); /* Compress and write nitems items of size size from buf to file, duplicating the interface of stdio's fwrite(), with size_t request and return types. If @@ -1465,7 +1468,7 @@ ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, is returned, and the error state is set to Z_STREAM_ERROR. */ -ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); +ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); /* Convert, format, compress, and write the arguments (...) to file under control of the string format, as in fprintf. gzprintf returns the number of @@ -1480,7 +1483,7 @@ ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); This can be determined using zlibCompileFlags(). */ -ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); +ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); /* Compress and write the given null-terminated string s to file, excluding the terminating null character. @@ -1488,7 +1491,7 @@ ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); gzputs returns the number of characters written, or -1 in case of error. */ -ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); +ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); /* Read and decompress bytes from file into buf, until len-1 characters are read, or until a newline character is read and transferred to buf, or an @@ -1502,13 +1505,13 @@ ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); buf are indeterminate. */ -ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); +ZEXTERN int ZEXPORT gzputc(gzFile file, int c); /* Compress and write c, converted to an unsigned char, into file. gzputc returns the value that was written, or -1 in case of error. */ -ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +ZEXTERN int ZEXPORT gzgetc(gzFile file); /* Read and decompress one byte from file. gzgetc returns this byte or -1 in case of end of file or error. This is implemented as a macro for speed. @@ -1517,7 +1520,7 @@ ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); points to has been clobbered or not. */ -ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); +ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); /* Push c back onto the stream for file to be read as the first character on the next read. At least one character of push-back is always allowed. @@ -1529,7 +1532,7 @@ ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); gzseek() or gzrewind(). */ -ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); +ZEXTERN int ZEXPORT gzflush(gzFile file, int flush); /* Flush all pending output to file. The parameter flush is as in the deflate() function. The return value is the zlib error number (see function @@ -1545,8 +1548,8 @@ ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); */ /* -ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, - z_off_t offset, int whence)); +ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, + z_off_t offset, int whence); Set the starting position to offset relative to whence for the next gzread or gzwrite on file. The offset represents a number of bytes in the @@ -1564,7 +1567,7 @@ ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, would be before the current position. */ -ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); +ZEXTERN int ZEXPORT gzrewind(gzFile file); /* Rewind file. This function is supported only for reading. @@ -1572,7 +1575,7 @@ ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); */ /* -ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); +ZEXTERN z_off_t ZEXPORT gztell(gzFile file); Return the starting position for the next gzread or gzwrite on file. This position represents a number of bytes in the uncompressed data stream, @@ -1583,7 +1586,7 @@ ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); */ /* -ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); +ZEXTERN z_off_t ZEXPORT gzoffset(gzFile file); Return the current compressed (actual) read or write offset of file. This offset includes the count of bytes that precede the gzip stream, for example @@ -1592,7 +1595,7 @@ ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); be used for a progress indicator. On error, gzoffset() returns -1. */ -ZEXTERN int ZEXPORT gzeof OF((gzFile file)); +ZEXTERN int ZEXPORT gzeof(gzFile file); /* Return true (1) if the end-of-file indicator for file has been set while reading, false (0) otherwise. Note that the end-of-file indicator is set @@ -1607,7 +1610,7 @@ ZEXTERN int ZEXPORT gzeof OF((gzFile file)); has grown since the previous end of file was detected. */ -ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); +ZEXTERN int ZEXPORT gzdirect(gzFile file); /* Return true (1) if file is being copied directly while reading, or false (0) if file is a gzip stream being decompressed. @@ -1628,7 +1631,7 @@ ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); gzip file reading and decompression, which may not be desired.) */ -ZEXTERN int ZEXPORT gzclose OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose(gzFile file); /* Flush all pending output for file, if necessary, close file and deallocate the (de)compression state. Note that once file is closed, you @@ -1641,8 +1644,8 @@ ZEXTERN int ZEXPORT gzclose OF((gzFile file)); last read ended in the middle of a gzip stream, or Z_OK on success. */ -ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); -ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_r(gzFile file); +ZEXTERN int ZEXPORT gzclose_w(gzFile file); /* Same as gzclose(), but gzclose_r() is only for use when reading, and gzclose_w() is only for use when writing or appending. The advantage to @@ -1653,7 +1656,7 @@ ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); zlib library. */ -ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); +ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum); /* Return the error message for the last error which occurred on file. errnum is set to zlib error number. If an error occurred in the file system @@ -1669,7 +1672,7 @@ ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); functions above that do not distinguish those cases in their return values. */ -ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); +ZEXTERN void ZEXPORT gzclearerr(gzFile file); /* Clear the error and end-of-file flags for file. This is analogous to the clearerr() function in stdio. This is useful for continuing to read a gzip @@ -1686,7 +1689,7 @@ ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); library. */ -ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); +ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len); /* Update a running Adler-32 checksum with the bytes buf[0..len-1] and return the updated checksum. An Adler-32 value is in the range of a 32-bit @@ -1706,15 +1709,15 @@ ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); if (adler != original_adler) error(); */ -ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, - z_size_t len)); +ZEXTERN uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, + z_size_t len); /* Same as adler32(), but with a size_t length. */ /* -ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, - z_off_t len2)); +ZEXTERN uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, + z_off_t len2); Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for @@ -1724,7 +1727,7 @@ ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, negative, the result has no meaning or utility. */ -ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); +ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len); /* Update a running CRC-32 with the bytes buf[0..len-1] and return the updated CRC-32. A CRC-32 value is in the range of a 32-bit unsigned integer. @@ -1742,30 +1745,30 @@ ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); if (crc != original_crc) error(); */ -ZEXTERN uLong ZEXPORT crc32_z OF((uLong crc, const Bytef *buf, - z_size_t len)); +ZEXTERN uLong ZEXPORT crc32_z(uLong crc, const Bytef *buf, + z_size_t len); /* Same as crc32(), but with a size_t length. */ /* -ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); +ZEXTERN uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2); Combine two CRC-32 check values into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, CRC-32 check values were calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and - len2. + len2. len2 must be non-negative. */ /* -ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t len2)); +ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t len2); Return the operator corresponding to length len2, to be used with - crc32_combine_op(). + crc32_combine_op(). len2 must be non-negative. */ -ZEXTERN uLong ZEXPORT crc32_combine_op OF((uLong crc1, uLong crc2, uLong op)); +ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op); /* Give the same result as crc32_combine(), using op in place of len2. op is is generated from len2 by crc32_combine_gen(). This will be faster than @@ -1778,20 +1781,20 @@ ZEXTERN uLong ZEXPORT crc32_combine_op OF((uLong crc1, uLong crc2, uLong op)); /* deflateInit and inflateInit are macros to allow checking the zlib version * and the compiler's view of z_stream: */ -ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, - const char *version, int stream_size)); -ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, - const char *version, int stream_size)); -ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, - int windowBits, int memLevel, - int strategy, const char *version, - int stream_size)); -ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, - const char *version, int stream_size)); -ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, - unsigned char FAR *window, - const char *version, - int stream_size)); +ZEXTERN int ZEXPORT deflateInit_(z_streamp strm, int level, + const char *version, int stream_size); +ZEXTERN int ZEXPORT inflateInit_(z_streamp strm, + const char *version, int stream_size); +ZEXTERN int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size); +ZEXTERN int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, + const char *version, int stream_size); +ZEXTERN int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size); #ifdef Z_PREFIX_SET # define z_deflateInit(strm, level) \ deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) @@ -1836,7 +1839,7 @@ struct gzFile_s { unsigned char *next; z_off64_t pos; }; -ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ +ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ @@ -1853,13 +1856,13 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ * without large file support, _LFS64_LARGEFILE must also be true */ #ifdef Z_LARGE64 - ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); - ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); - ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); - ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); - ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); - ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); - ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off64_t)); + ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); + ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int); + ZEXTERN z_off64_t ZEXPORT gztell64(gzFile); + ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t); #endif #if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) @@ -1881,50 +1884,50 @@ ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ # define crc32_combine_gen crc32_combine_gen64 # endif # ifndef Z_LARGE64 - ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); - ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); - ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); - ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); - ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t)); + ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); + ZEXTERN z_off_t ZEXPORT gzseek64(gzFile, z_off_t, int); + ZEXTERN z_off_t ZEXPORT gztell64(gzFile); + ZEXTERN z_off_t ZEXPORT gzoffset64(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); # endif #else - ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); - ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); - ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); - ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); - ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); + ZEXTERN gzFile ZEXPORT gzopen(const char *, const char *); + ZEXTERN z_off_t ZEXPORT gzseek(gzFile, z_off_t, int); + ZEXTERN z_off_t ZEXPORT gztell(gzFile); + ZEXTERN z_off_t ZEXPORT gzoffset(gzFile); + ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); #endif #else /* Z_SOLO */ - ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); + ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); #endif /* !Z_SOLO */ /* undocumented functions */ -ZEXTERN const char * ZEXPORT zError OF((int)); -ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); -ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); -ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); -ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); -ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF((z_streamp)); -ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); -ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); +ZEXTERN const char * ZEXPORT zError(int); +ZEXTERN int ZEXPORT inflateSyncPoint(z_streamp); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table(void); +ZEXTERN int ZEXPORT inflateUndermine(z_streamp, int); +ZEXTERN int ZEXPORT inflateValidate(z_streamp, int); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed(z_streamp); +ZEXTERN int ZEXPORT inflateResetKeep(z_streamp); +ZEXTERN int ZEXPORT deflateResetKeep(z_streamp); #if defined(_WIN32) && !defined(Z_SOLO) -ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, - const char *mode)); +ZEXTERN gzFile ZEXPORT gzopen_w(const wchar_t *path, + const char *mode); #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifndef Z_SOLO -ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, - const char *format, - va_list va)); +ZEXTERN int ZEXPORTVA gzvprintf(gzFile file, + const char *format, + va_list va); # endif #endif diff --git a/src/native/external/zlib/zlib2ansi b/src/native/external/zlib/zlib2ansi deleted file mode 100644 index 23b2a1d5a3e..00000000000 --- a/src/native/external/zlib/zlib2ansi +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/perl - -# Transform K&R C function definitions into ANSI equivalent. -# -# Author: Paul Marquess -# Version: 1.0 -# Date: 3 October 2006 - -# TODO -# -# Assumes no function pointer parameters. unless they are typedefed. -# Assumes no literal strings that look like function definitions -# Assumes functions start at the beginning of a line - -use strict; -use warnings; - -local $/; -$_ = <>; - -my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments - -my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; -my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; -my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; - - -while (s/^ - ( # Start $1 - ( # Start $2 - .*? # Minimal eat content - ( ^ \w [\w\s\*]+ ) # $3 -- function name - \s* # optional whitespace - ) # $2 - Matched up to before parameter list - - \( \s* # Literal "(" + optional whitespace - ( [^\)]+ ) # $4 - one or more anythings except ")" - \s* \) # optional whitespace surrounding a Literal ")" - - ( (?: $dList )+ ) # $5 - - $sp ^ { # literal "{" at start of line - ) # Remember to $1 - //xsom - ) -{ - my $all = $1 ; - my $prefix = $2; - my $param_list = $4 ; - my $params = $5; - - StripComments($params); - StripComments($param_list); - $param_list =~ s/^\s+//; - $param_list =~ s/\s+$//; - - my $i = 0 ; - my %pList = map { $_ => $i++ } - split /\s*,\s*/, $param_list; - my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; - - my @params = split /\s*;\s*/, $params; - my @outParams = (); - foreach my $p (@params) - { - if ($p =~ /,/) - { - my @bits = split /\s*,\s*/, $p; - my $first = shift @bits; - $first =~ s/^\s*//; - push @outParams, $first; - $first =~ /^(\w+\s*)/; - my $type = $1 ; - push @outParams, map { $type . $_ } @bits; - } - else - { - $p =~ s/^\s+//; - push @outParams, $p; - } - } - - - my %tmp = map { /$pMatch/; $_ => $pList{$1} } - @outParams ; - - @outParams = map { " $_" } - sort { $tmp{$a} <=> $tmp{$b} } - @outParams ; - - print $prefix ; - print "(\n" . join(",\n", @outParams) . ")\n"; - print "{" ; - -} - -# Output any trailing code. -print ; -exit 0; - - -sub StripComments -{ - - no warnings; - - # Strip C & C++ comments - # From the perlfaq - $_[0] =~ - - s{ - /\* ## Start of /* ... */ comment - [^*]*\*+ ## Non-* followed by 1-or-more *'s - ( - [^/*][^*]*\*+ - )* ## 0-or-more things which don't start with / - ## but do end with '*' - / ## End of /* ... */ comment - - | ## OR C++ Comment - // ## Start of C++ comment // - [^\n]* ## followed by 0-or-more non end of line characters - - | ## OR various things which aren't comments: - - ( - " ## Start of " ... " string - ( - \\. ## Escaped char - | ## OR - [^"\\] ## Non "\ - )* - " ## End of " ... " string - - | ## OR - - ' ## Start of ' ... ' string - ( - \\. ## Escaped char - | ## OR - [^'\\] ## Non '\ - )* - ' ## End of ' ... ' string - - | ## OR - - . ## Anything other char - [^/"'\\]* ## Chars which doesn't start a comment, string or escape - ) - }{$2}gxs; - -} diff --git a/src/native/external/zlib/zutil.c b/src/native/external/zlib/zutil.c index 9543ae825e3..b1c5d2d3c6d 100644 --- a/src/native/external/zlib/zutil.c +++ b/src/native/external/zlib/zutil.c @@ -24,13 +24,11 @@ z_const char * const z_errmsg[10] = { }; -const char * ZEXPORT zlibVersion() -{ +const char * ZEXPORT zlibVersion(void) { return ZLIB_VERSION; } -uLong ZEXPORT zlibCompileFlags() -{ +uLong ZEXPORT zlibCompileFlags(void) { uLong flags; flags = 0; @@ -121,9 +119,7 @@ uLong ZEXPORT zlibCompileFlags() # endif int ZLIB_INTERNAL z_verbose = verbose; -void ZLIB_INTERNAL z_error(m) - char *m; -{ +void ZLIB_INTERNAL z_error(char *m) { fprintf(stderr, "%s\n", m); exit(1); } @@ -132,9 +128,7 @@ void ZLIB_INTERNAL z_error(m) /* exported to allow conversion of error code to string for compress() and * uncompress() */ -const char * ZEXPORT zError(err) - int err; -{ +const char * ZEXPORT zError(int err) { return ERR_MSG(err); } @@ -148,22 +142,14 @@ const char * ZEXPORT zError(err) #ifndef HAVE_MEMCPY -void ZLIB_INTERNAL zmemcpy(dest, source, len) - Bytef* dest; - const Bytef* source; - uInt len; -{ +void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len) { if (len == 0) return; do { *dest++ = *source++; /* ??? to be unrolled */ } while (--len != 0); } -int ZLIB_INTERNAL zmemcmp(s1, s2, len) - const Bytef* s1; - const Bytef* s2; - uInt len; -{ +int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) { uInt j; for (j = 0; j < len; j++) { @@ -172,10 +158,7 @@ int ZLIB_INTERNAL zmemcmp(s1, s2, len) return 0; } -void ZLIB_INTERNAL zmemzero(dest, len) - Bytef* dest; - uInt len; -{ +void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len) { if (len == 0) return; do { *dest++ = 0; /* ??? to be unrolled */ @@ -216,8 +199,7 @@ local ptr_table table[MAX_PTR]; * a protected system like OS/2. Use Microsoft C instead. */ -voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) -{ +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { voidpf buf; ulg bsize = (ulg)items*size; @@ -242,8 +224,7 @@ voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) return buf; } -void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) -{ +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { int n; (void)opaque; @@ -279,14 +260,12 @@ void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) # define _hfree hfree #endif -voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) -{ +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) { (void)opaque; return _halloc((long)items, size); } -void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) -{ +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { (void)opaque; _hfree(ptr); } @@ -299,25 +278,18 @@ void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) #ifndef MY_ZCALLOC /* Any system without a special alloc function */ #ifndef STDC -extern voidp malloc OF((uInt size)); -extern voidp calloc OF((uInt items, uInt size)); -extern void free OF((voidpf ptr)); +extern voidp malloc(uInt size); +extern voidp calloc(uInt items, uInt size); +extern void free(voidpf ptr); #endif -voidpf ZLIB_INTERNAL zcalloc(opaque, items, size) - voidpf opaque; - unsigned items; - unsigned size; -{ +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } -void ZLIB_INTERNAL zcfree(opaque, ptr) - voidpf opaque; - voidpf ptr; -{ +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { (void)opaque; free(ptr); } diff --git a/src/native/external/zlib/zutil.h b/src/native/external/zlib/zutil.h index 0bc7f4ecd1c..48dd7febae6 100644 --- a/src/native/external/zlib/zutil.h +++ b/src/native/external/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2022 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -56,7 +56,7 @@ typedef unsigned long ulg; extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ -#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] +#define ERR_MSG(err) z_errmsg[(err) < -6 || (err) > 2 ? 9 : 2 - (err)] #define ERR_RETURN(strm,err) \ return (strm->msg = ERR_MSG(err), (err)) @@ -137,17 +137,8 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#if defined(MACOS) || defined(TARGET_OS_MAC) +#if defined(MACOS) # define OS_CODE 7 -# ifndef Z_SOLO -# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os -# include /* for fdopen */ -# else -# ifndef fdopen -# define fdopen(fd,mode) NULL /* No fdopen() */ -# endif -# endif -# endif #endif #ifdef __acorn @@ -170,18 +161,6 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define OS_CODE 19 #endif -#if defined(_BEOS_) || defined(RISCOS) -# define fdopen(fd,mode) NULL /* No fdopen() */ -#endif - -#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX -# if defined(_WIN32_WCE) -# define fdopen(fd,mode) NULL /* No fdopen() */ -# else -# define fdopen(fd,type) _fdopen(fd,type) -# endif -#endif - #if defined(__BORLANDC__) && !defined(MSDOS) #pragma warn -8004 #pragma warn -8008 @@ -191,9 +170,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* provide prototypes for these when building zlib without LFS */ #if !defined(_WIN32) && \ (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) - ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); - ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t)); + ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); + ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); #endif /* common defaults */ @@ -232,16 +211,16 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define zmemzero(dest, len) memset(dest, 0, len) # endif #else - void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); - int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); - void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); + void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len); + int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len); + void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len); #endif /* Diagnostic functions */ #ifdef ZLIB_DEBUG # include extern int ZLIB_INTERNAL z_verbose; - extern void ZLIB_INTERNAL z_error OF((char *m)); + extern void ZLIB_INTERNAL z_error(char *m); # define Assert(cond,msg) {if(!(cond)) z_error(msg);} # define Trace(x) {if (z_verbose>=0) fprintf x ;} # define Tracev(x) {if (z_verbose>0) fprintf x ;} @@ -258,9 +237,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifndef Z_SOLO - voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, - unsigned size)); - void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); + voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, + unsigned size); + void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr); #endif #define ZALLOC(strm, items, size) \ From 250354be83584ba28e9f50c39ce06348408782a3 Mon Sep 17 00:00:00 2001 From: Radek Zikmund <32671551+rzikm@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:39:35 +0100 Subject: [PATCH 033/151] Fix failing CertificateValidationRemoteServer.ConnectWithRevocation_WithCallback test (#99950) * Don't add empty Organization to Subject string * Make sure testName is populated when creating test PKI * Minor changes --- .../X509Certificates/CertificateAuthority.cs | 10 ++++------ .../CertificateValidationRemoteServer.cs | 5 ++++- .../SslStreamCertificateContextTests.cs | 1 + 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libraries/Common/tests/System/Security/Cryptography/X509Certificates/CertificateAuthority.cs b/src/libraries/Common/tests/System/Security/Cryptography/X509Certificates/CertificateAuthority.cs index 184d8a62e99..beb32116171 100644 --- a/src/libraries/Common/tests/System/Security/Cryptography/X509Certificates/CertificateAuthority.cs +++ b/src/libraries/Common/tests/System/Security/Cryptography/X509Certificates/CertificateAuthority.cs @@ -179,7 +179,7 @@ internal X509Certificate2 CreateOcspSigner(string subject, RSA publicKey) subject, publicKey, TimeSpan.FromSeconds(1), - new X509ExtensionCollection() { s_eeConstraints, s_eeKeyUsage, s_ocspResponderEku}, + new X509ExtensionCollection() { s_eeConstraints, s_eeKeyUsage, s_ocspResponderEku }, ocspResponder: true); } @@ -950,12 +950,10 @@ private static string BuildSubject( PkiOptions pkiOptions, bool includePkiOptions) { - if (includePkiOptions) - { - return $"CN=\"{cn}\", O=\"{testName}\", OU=\"{pkiOptions}\""; - } + string testNamePart = !string.IsNullOrWhiteSpace(testName) ? $", O=\"{testName}\"" : ""; + string pkiOptionsPart = includePkiOptions ? $", OU=\"{pkiOptions}\"" : ""; - return $"CN=\"{cn}\", O=\"{testName}\""; + return $"CN=\"{cn}\"" + testNamePart + pkiOptionsPart; } } } diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/CertificateValidationRemoteServer.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/CertificateValidationRemoteServer.cs index e804849dd7f..35cefd116ac 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/CertificateValidationRemoteServer.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/CertificateValidationRemoteServer.cs @@ -7,6 +7,7 @@ using System.Net.Sockets; using System.Net.Test.Common; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; using System.Security.Cryptography.X509Certificates.Tests.Common; @@ -188,7 +189,8 @@ static bool CertificateValidationCallback( private async Task ConnectWithRevocation_WithCallback_Core( X509RevocationMode revocationMode, bool? offlineContext = false, - bool noIntermediates = false) + bool noIntermediates = false, + [CallerMemberName] string testName = null) { string offlinePart = offlineContext.HasValue ? offlineContext.GetValueOrDefault().ToString().ToLower() : "null"; string serverName = $"{revocationMode.ToString().ToLower()}.{offlinePart}.server.example"; @@ -201,6 +203,7 @@ private async Task ConnectWithRevocation_WithCallback_Core( out CertificateAuthority rootAuthority, out CertificateAuthority[] intermediateAuthorities, out X509Certificate2 serverCert, + testName: testName, intermediateAuthorityCount: noIntermediates ? 0 : 1, subjectName: serverName, keySize: 2048, diff --git a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamCertificateContextTests.cs b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamCertificateContextTests.cs index a3fd09c41ef..69cb644243a 100644 --- a/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamCertificateContextTests.cs +++ b/src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamCertificateContextTests.cs @@ -24,6 +24,7 @@ public static async Task Create_OcspDoesNotReturnOrCacheInvalidStapleData() out CertificateAuthority rootAuthority, out CertificateAuthority[] intermediateAuthorities, out X509Certificate2 serverCert, + testName: nameof(Create_OcspDoesNotReturnOrCacheInvalidStapleData), intermediateAuthorityCount: 1, subjectName: serverName, keySize: 2048, From afb3fcad64be34154300264a65ff324208e76d32 Mon Sep 17 00:00:00 2001 From: Matous Kozak <55735845+matouskozak@users.noreply.github.com> Date: Wed, 20 Mar 2024 18:01:57 +0100 Subject: [PATCH 034/151] [mono] Stop exporting ICU symbols from Mono (#99449) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * do not export ICU symbols from Mono (libmonosgen-2.0.dylib and libSystem.Globalization.Native.dylib) --------- Co-authored-by: Alexander Köplinger --- src/mono/mono/mini/CMakeLists.txt | 10 +++++++++- .../libs/System.Globalization.Native/CMakeLists.txt | 10 +++++++++- src/tasks/AppleAppBuilder/Xcode.cs | 11 ++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/mono/mono/mini/CMakeLists.txt b/src/mono/mono/mini/CMakeLists.txt index 5d6ef3dfa3c..08b90567eb2 100644 --- a/src/mono/mono/mini/CMakeLists.txt +++ b/src/mono/mono/mini/CMakeLists.txt @@ -421,7 +421,15 @@ if(NOT DISABLE_SHARED_LIBS) # to avoid a conflict we rename the import library with the .import.lib suffix set_target_properties(monosgen-shared PROPERTIES IMPORT_SUFFIX ".import.lib") endif() - target_link_libraries(monosgen-shared PRIVATE ${OS_LIBS} ${LLVM_LIBS} ${ICU_LIBS} ${Z_LIBS}) + if (CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS) + target_link_libraries(monosgen-shared PRIVATE ${OS_LIBS} ${LLVM_LIBS} ${Z_LIBS}) + add_linker_flag(-Wl,-L${ICU_LIBDIR}) + add_linker_flag(-Wl,-hidden-licuuc) + add_linker_flag(-Wl,-hidden-licui18n) + add_linker_flag(-Wl,-hidden-licudata) + else() + target_link_libraries(monosgen-shared PRIVATE ${OS_LIBS} ${LLVM_LIBS} ${ICU_LIBS} ${Z_LIBS}) + endif() if(ICU_LDFLAGS) set_property(TARGET monosgen-shared APPEND_STRING PROPERTY LINK_FLAGS " ${ICU_LDFLAGS}") endif() diff --git a/src/native/libs/System.Globalization.Native/CMakeLists.txt b/src/native/libs/System.Globalization.Native/CMakeLists.txt index 0545ace30f4..1fcaf0b6bd5 100644 --- a/src/native/libs/System.Globalization.Native/CMakeLists.txt +++ b/src/native/libs/System.Globalization.Native/CMakeLists.txt @@ -66,7 +66,15 @@ set(NATIVEGLOBALIZATION_SOURCES if (DEFINED CMAKE_ICU_DIR) include_directories(${CMAKE_ICU_DIR}/include) - link_libraries(${CMAKE_ICU_DIR}/lib/libicuuc.a ${CMAKE_ICU_DIR}/lib/libicui18n.a ${CMAKE_ICU_DIR}/lib/libicudata.a) + if (CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS) + add_linker_flag(-Wl,-L${CMAKE_ICU_DIR}/lib) + add_linker_flag(-Wl,-hidden-licuuc) + add_linker_flag(-Wl,-hidden-licui18n) + add_linker_flag(-Wl,-hidden-licudata) + else() + link_libraries(${CMAKE_ICU_DIR}/lib/libicuuc.a ${CMAKE_ICU_DIR}/lib/libicui18n.a ${CMAKE_ICU_DIR}/lib/libicudata.a) + endif() + link_libraries(stdc++) endif() diff --git a/src/tasks/AppleAppBuilder/Xcode.cs b/src/tasks/AppleAppBuilder/Xcode.cs index 0e05d916842..f253b41a2b1 100644 --- a/src/tasks/AppleAppBuilder/Xcode.cs +++ b/src/tasks/AppleAppBuilder/Xcode.cs @@ -396,9 +396,14 @@ public string GenerateCMake( } else if (forceAOT || !(preferDylibs && dylibExists)) { - // these libraries are pinvoked - // -force_load will be removed once we enable direct-pinvokes for AOT - toLink += $" \"-force_load {lib}\"{Environment.NewLine}"; + // do not export symbols from ICU libraries + if (libName == "libicui18n" || libName == "libicudata" || libName == "libicuuc") { + toLink += $" \"-load_hidden {lib}\"{Environment.NewLine}"; + } else { + // these libraries are pinvoked + // -force_load will be removed once we enable direct-pinvokes for AOT + toLink += $" \"-force_load {lib}\"{Environment.NewLine}"; + } } } From 99c7022f7644866770c4a3bd6a73d4e27922097c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:50:28 +0100 Subject: [PATCH 035/151] [release/8.0-staging] Fix AV in HttpTelemetry.WriteEvent (#99607) * Fix AV in HttpTelemetry.WriteEvent * Move descrs into fixed scope --------- Co-authored-by: Miha Zupan --- .../src/System/Net/Http/HttpTelemetry.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs b/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs index 9989bc4568e..3dfc789919b 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs @@ -339,13 +339,13 @@ private unsafe void WriteEvent(int eventId, byte arg1, byte arg2, long arg3, str arg5 ??= ""; arg7 ??= ""; - const int NumEventDatas = 7; - EventData* descrs = stackalloc EventData[NumEventDatas]; - fixed (char* arg4Ptr = arg4) fixed (char* arg5Ptr = arg5) fixed (char* arg7Ptr = arg7) { + const int NumEventDatas = 7; + EventData* descrs = stackalloc EventData[NumEventDatas]; + descrs[0] = new EventData { DataPointer = (IntPtr)(&arg1), @@ -381,9 +381,9 @@ private unsafe void WriteEvent(int eventId, byte arg1, byte arg2, long arg3, str DataPointer = (IntPtr)arg7Ptr, Size = (arg7.Length + 1) * sizeof(char) }; - } - WriteEventCore(eventId, NumEventDatas, descrs); + WriteEventCore(eventId, NumEventDatas, descrs); + } } } } From b7f047793b9377bd01b51421d8e5740af97c0d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20P=C3=ADchov=C3=A1?= <11718369+ManickaP@users.noreply.github.com> Date: Tue, 26 Mar 2024 17:54:03 +0100 Subject: [PATCH 036/151] [Http/2] Fix handling of effectively empty DATA frame (#99502) (#99677) * Fix handling effectively empty DATA frame * Added test --- .../SocketsHttpHandler/Http2Connection.cs | 6 ++- .../HttpClientHandlerTest.Http2.cs | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index e9cce1c24d3..5b795f0a884 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -765,12 +765,14 @@ private void ProcessDataFrame(FrameHeader frameHeader) // Just ignore the frame in this case. ReadOnlySpan frameData = GetFrameData(_incomingBuffer.ActiveSpan.Slice(0, frameHeader.PayloadLength), hasPad: frameHeader.PaddedFlag, hasPriority: false); - if (http2Stream != null) { bool endStream = frameHeader.EndStreamFlag; - http2Stream.OnResponseData(frameData, endStream); + if (frameData.Length > 0 || endStream) + { + http2Stream.OnResponseData(frameData, endStream); + } if (!endStream && frameData.Length > 0) { diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs index 25a3dcd7c69..07f19ef3029 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs @@ -204,6 +204,50 @@ public async Task Http2_ZeroLengthResponseBody_Success() } } + [Fact] + public async Task Http2_DataFrameOnlyPadding_Success() + { + using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) + using (HttpClient client = CreateHttpClient()) + { + Task sendTask = client.GetAsync(server.Address, HttpCompletionOption.ResponseHeadersRead); + + Http2LoopbackConnection connection = await server.EstablishConnectionAsync(); + + int streamId = await connection.ReadRequestHeaderAsync(); + + await connection.SendDefaultResponseHeadersAsync(streamId); + + // Send zero-length DATA frame with padding + byte paddingLength = byte.MaxValue; + int dataLength = 1024; + DataFrame frame = new DataFrame(new byte[0], FrameFlags.Padded, paddingLength, streamId); + await connection.WriteFrameAsync(frame); + + HttpResponseMessage response = await sendTask; + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + using var responseStream = response.Content.ReadAsStream(); + + // The read must pend because we havent received any data yet. + var buffer = new byte[dataLength]; + var readTask = responseStream.ReadAtLeastAsync(buffer, dataLength); + Assert.False(readTask.IsCompleted); + + // Send DATA frame with padding + frame = new DataFrame(new byte[dataLength], FrameFlags.Padded, paddingLength, streamId); + await connection.WriteFrameAsync(frame); + + Assert.Equal(dataLength, await readTask); + + // Send zero-length, end-stream DATA frame with padding + frame = new DataFrame(new byte[0], FrameFlags.Padded | FrameFlags.EndStream, paddingLength, streamId); + await connection.WriteFrameAsync(frame); + + Assert.Equal(0, await responseStream.ReadAsync(buffer)); + } + } + [Theory] [InlineData("Client content", null)] [InlineData("Client content", "Server content")] From 3f774aa658f3963f43876846cb439dbb9eba8418 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 14:38:30 +0100 Subject: [PATCH 037/151] [release/8.0-staging] [HttpStress] [SslStress] Workaround image bug in 1es-windows-2022-open (#100303) Backport of #100145 to release/8.0-staging --- eng/pipelines/libraries/stress/http.yml | 3 +++ eng/pipelines/libraries/stress/ssl.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/eng/pipelines/libraries/stress/http.yml b/eng/pipelines/libraries/stress/http.yml index f4f9c45de36..68bfcef6c50 100644 --- a/eng/pipelines/libraries/stress/http.yml +++ b/eng/pipelines/libraries/stress/http.yml @@ -119,6 +119,9 @@ extends: lfs: false - powershell: | + # Workaround for https://github.com/microsoft/azure-pipelines-agent/issues/4554. Undo when the image bug is fixed. + Remove-Item -Path "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\Microsoft.VCToolsVersion.v143.default.txt" + $(dockerfilesFolder)/build-docker-sdk.ps1 -w -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION) echo "##vso[task.setvariable variable=succeeded;isOutput=true]true" name: buildRuntime diff --git a/eng/pipelines/libraries/stress/ssl.yml b/eng/pipelines/libraries/stress/ssl.yml index ab93994400d..a70a18e828f 100644 --- a/eng/pipelines/libraries/stress/ssl.yml +++ b/eng/pipelines/libraries/stress/ssl.yml @@ -76,6 +76,9 @@ extends: lfs: false - powershell: | + # Workaround for https://github.com/microsoft/azure-pipelines-agent/issues/4554. Undo when the image bug is fixed. + Remove-Item -Path "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\Microsoft.VCToolsVersion.v143.default.txt" + $(dockerfilesFolder)/build-docker-sdk.ps1 -w -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION) displayName: Build CLR and Libraries From ab416e9ef39c6709b815e6ca31753a4c6e44ed58 Mon Sep 17 00:00:00 2001 From: Omair Majid Date: Wed, 27 Mar 2024 15:53:19 -0400 Subject: [PATCH 038/151] Support building against clang 18 (#100258) This is a targeted backport from a few other PRs that makes it possible to build dotnet/runtme's 8.0 branch on Fedora 40 which includes clang 18. - https://github.com/dotnet/arcade/pull/14572 - https://github.com/dotnet/runtime/pull/94782 - https://github.com/dotnet/runtime/pull/99811 --- eng/common/native/init-compiler.sh | 2 +- eng/native/configurecompiler.cmake | 3 +++ src/coreclr/pal/src/include/pal/palinternal.h | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index f5c1ec7eafe..2d5660642b8 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index 18381101853..0e6ee88b245 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -590,6 +590,9 @@ if (CLR_CMAKE_HOST_UNIX) # other clang 16.0 suppressions add_compile_options(-Wno-single-bit-bitfield-constant-conversion) add_compile_options(-Wno-cast-function-type-strict) + + # clang 18.1 supressions + add_compile_options(-Wno-switch-default) else() add_compile_options(-Wno-uninitialized) add_compile_options(-Wno-strict-aliasing) diff --git a/src/coreclr/pal/src/include/pal/palinternal.h b/src/coreclr/pal/src/include/pal/palinternal.h index a7c5ba129c9..3b8a55a9449 100644 --- a/src/coreclr/pal/src/include/pal/palinternal.h +++ b/src/coreclr/pal/src/include/pal/palinternal.h @@ -426,6 +426,7 @@ function_name() to call the system's implementation #undef va_start #undef va_end #undef va_copy +#undef va_arg #undef stdin #undef stdout #undef stderr From 51c9e0b09539f4e2213306abb6c0d8d9b48a050f Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 14 Mar 2024 10:49:38 -0700 Subject: [PATCH 039/151] Ensure Number.BigInteger doesn't try to access an invalid block --- .../src/System/Number.BigInteger.cs | 104 +++++++++++++++--- 1 file changed, 88 insertions(+), 16 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs index aba4b814fc3..e8b38321185 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs @@ -358,11 +358,24 @@ public static void Add(scoped ref BigInteger lhs, scoped ref BigInteger rhs, out resultIndex++; } + int resultLength = largeLength; + // If there's still a carry, append a new block if (carry != 0) { Debug.Assert(carry == 1); - Debug.Assert((resultIndex == largeLength) && (largeLength < MaxBlockCount)); + Debug.Assert(resultIndex == resultLength); + Debug.Assert(unchecked((uint)(resultLength)) < MaxBlockCount); + + if (unchecked((uint)(resultLength)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out result); + return; + } result._blocks[resultIndex] = 1; result._length++; @@ -724,6 +737,8 @@ public static void Multiply(scoped ref BigInteger lhs, uint value, out BigIntege int index = 0; uint carry = 0; + int resultLength = lhsLength; + while (index < lhsLength) { ulong product = ((ulong)(lhs._blocks[index]) * value) + carry; @@ -735,14 +750,23 @@ public static void Multiply(scoped ref BigInteger lhs, uint value, out BigIntege if (carry != 0) { - Debug.Assert(unchecked((uint)(lhsLength)) + 1 <= MaxBlockCount); + Debug.Assert(unchecked((uint)(resultLength)) < MaxBlockCount); + + if (unchecked((uint)(resultLength)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out result); + return; + } + result._blocks[index] = carry; - result._length = (lhsLength + 1); - } - else - { - result._length = lhsLength; + resultLength += 1; } + + result._length = resultLength; } public static void Multiply(scoped ref BigInteger lhs, scoped ref BigInteger rhs, out BigInteger result) @@ -777,6 +801,16 @@ public static void Multiply(scoped ref BigInteger lhs, scoped ref BigInteger rhs int maxResultLength = smallLength + largeLength; Debug.Assert(unchecked((uint)(maxResultLength)) <= MaxBlockCount); + if (unchecked((uint)(maxResultLength)) > MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out result); + return; + } + // Zero out result internal blocks. result._length = maxResultLength; result.Clear((uint)maxResultLength); @@ -822,7 +856,19 @@ public static void Pow2(uint exponent, out BigInteger result) { uint blocksToShift = DivRem32(exponent, out uint remainingBitsToShift); result._length = (int)blocksToShift + 1; + Debug.Assert(unchecked((uint)result._length) <= MaxBlockCount); + + if (unchecked((uint)result._length) > MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out result); + return; + } + if (blocksToShift > 0) { result.Clear(blocksToShift); @@ -1012,7 +1058,18 @@ public void Add(uint value) } } - Debug.Assert(unchecked((uint)(length)) + 1 <= MaxBlockCount); + Debug.Assert(unchecked((uint)(length)) < MaxBlockCount); + + if (unchecked((uint)(length)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out this); + return; + } + _blocks[length] = 1; _length = length + 1; } @@ -1074,9 +1131,20 @@ public void Multiply10() if (carry != 0) { - Debug.Assert(unchecked((uint)(_length)) + 1 <= MaxBlockCount); + Debug.Assert(unchecked((uint)(length)) < MaxBlockCount); + + if (unchecked((uint)(length)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out this); + return; + } + _blocks[index] = (uint)carry; - _length++; + _length = length + 1; } } @@ -1149,11 +1217,19 @@ public void ShiftLeft(uint shift) int readIndex = (length - 1); int writeIndex = readIndex + (int)(blocksToShift); + uint remainingBitsInLastBlock = (uint)BitOperations.LeadingZeroCount(_blocks[readIndex]); + + if (remainingBitsToShift > remainingBitsInLastBlock) + { + // We need an extra block for the partial shift + writeIndex++; + } + + Debug.Assert(unchecked((uint)(writeIndex)) < MaxBlockCount); + // Check if the shift is block aligned if (remainingBitsToShift == 0) { - Debug.Assert(writeIndex < MaxBlockCount); - while (readIndex >= 0) { _blocks[writeIndex] = _blocks[readIndex]; @@ -1168,10 +1244,6 @@ public void ShiftLeft(uint shift) } else { - // We need an extra block for the partial shift - writeIndex++; - Debug.Assert(writeIndex < MaxBlockCount); - // Set the length to hold the shifted blocks _length = writeIndex + 1; From aa046223d94654213cdec4b103bfcb31b6a79385 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Fri, 15 Mar 2024 09:55:57 -0700 Subject: [PATCH 040/151] Fixing the shift-left handling to correctly account for overshifting --- .../src/System/Number.BigInteger.cs | 48 +++++++++++++------ 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs index e8b38321185..7cb8578fd63 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.BigInteger.cs @@ -31,7 +31,9 @@ internal unsafe ref struct BigInteger private const int MaxBits = BitsForLongestBinaryMantissa + BitsForLongestDigitSequence + BitsPerBlock; private const int BitsPerBlock = sizeof(int) * 8; - private const int MaxBlockCount = (MaxBits + (BitsPerBlock - 1)) / BitsPerBlock; + + // We need one extra block to make our shift left algorithm significantly simpler + private const int MaxBlockCount = ((MaxBits + (BitsPerBlock - 1)) / BitsPerBlock) + 1; private static ReadOnlySpan Pow10UInt32Table => new uint[] { @@ -302,7 +304,8 @@ internal unsafe ref struct BigInteger 0xD9D61A05, 0x00000325, - // 9 Trailing blocks to ensure MaxBlockCount + // 10 Trailing blocks to ensure MaxBlockCount + 0x00000000, 0x00000000, 0x00000000, 0x00000000, @@ -737,8 +740,6 @@ public static void Multiply(scoped ref BigInteger lhs, uint value, out BigIntege int index = 0; uint carry = 0; - int resultLength = lhsLength; - while (index < lhsLength) { ulong product = ((ulong)(lhs._blocks[index]) * value) + carry; @@ -748,6 +749,8 @@ public static void Multiply(scoped ref BigInteger lhs, uint value, out BigIntege index++; } + int resultLength = lhsLength; + if (carry != 0) { Debug.Assert(unchecked((uint)(resultLength)) < MaxBlockCount); @@ -1217,19 +1220,21 @@ public void ShiftLeft(uint shift) int readIndex = (length - 1); int writeIndex = readIndex + (int)(blocksToShift); - uint remainingBitsInLastBlock = (uint)BitOperations.LeadingZeroCount(_blocks[readIndex]); - - if (remainingBitsToShift > remainingBitsInLastBlock) - { - // We need an extra block for the partial shift - writeIndex++; - } - - Debug.Assert(unchecked((uint)(writeIndex)) < MaxBlockCount); - // Check if the shift is block aligned if (remainingBitsToShift == 0) { + Debug.Assert(unchecked((uint)(length)) < MaxBlockCount); + + if (unchecked((uint)(length)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out this); + return; + } + while (readIndex >= 0) { _blocks[writeIndex] = _blocks[readIndex]; @@ -1244,6 +1249,21 @@ public void ShiftLeft(uint shift) } else { + // We need an extra block for the partial shift + + writeIndex++; + Debug.Assert(unchecked((uint)(length)) < MaxBlockCount); + + if (unchecked((uint)(length)) >= MaxBlockCount) + { + // We shouldn't reach here, and the above assert will help flag this + // during testing, but we'll ensure that we return a safe value of + // zero in the case we end up overflowing in any way. + + SetZero(out this); + return; + } + // Set the length to hold the shifted blocks _length = writeIndex + 1; From f36dd88c5ef2e59e46ca60a493cdacb0c7657129 Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:32:50 +0100 Subject: [PATCH 041/151] [release/8.0][browser] WebSocket works differently depending on if we look up its state or not (#99673) * Fix. * Missing change - enqueue promises even when socket is closed. * More tests. --- .../Handlers/EchoWebSocketHandler.cs | 12 +++++ .../BrowserWebSockets/BrowserInterop.cs | 8 ++-- .../BrowserWebSockets/BrowserWebSocket.cs | 6 --- .../tests/CloseTest.cs | 46 ++++++++++++++++++- src/mono/wasm/runtime/exports-internal.ts | 3 +- src/mono/wasm/runtime/web-socket.ts | 14 +++++- 6 files changed, 75 insertions(+), 14 deletions(-) diff --git a/src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/EchoWebSocketHandler.cs b/src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/EchoWebSocketHandler.cs index 8304f2d1156..a290ce63bd4 100644 --- a/src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/EchoWebSocketHandler.cs +++ b/src/libraries/Common/tests/System/Net/Prerequisites/NetCoreServer/Handlers/EchoWebSocketHandler.cs @@ -144,6 +144,18 @@ await socket.CloseAsync( { await Task.Delay(5000); } + else if (receivedMessage == ".receiveMessageAfterClose") + { + byte[] buffer = new byte[1024]; + string message = $"{receivedMessage} {DateTime.Now.ToString("HH:mm:ss")}"; + buffer = System.Text.Encoding.UTF8.GetBytes(message); + await socket.SendAsync( + new ArraySegment(buffer, 0, message.Length), + WebSocketMessageType.Text, + true, + CancellationToken.None); + await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, receivedMessage, CancellationToken.None); + } else if (socket.State == WebSocketState.Open) { sendMessage = true; diff --git a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs index 5ea45c2a7a2..e535924a015 100644 --- a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs +++ b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs @@ -19,11 +19,13 @@ internal static partial class BrowserInterop public static int GetReadyState(JSObject? webSocket) { if (webSocket == null || webSocket.IsDisposed) return -1; - int? readyState = webSocket.GetPropertyAsInt32("readyState"); - if (!readyState.HasValue) return -1; - return readyState.Value; + return BrowserInterop.WebSocketGetState(webSocket); } + [JSImport("INTERNAL.ws_get_state")] + public static partial int WebSocketGetState( + JSObject webSocket); + [JSImport("INTERNAL.ws_wasm_create")] public static partial JSObject WebSocketCreate( string uri, diff --git a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs index 879d45ca0da..6c8f7a036ed 100644 --- a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs +++ b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs @@ -385,12 +385,6 @@ private void CreateCore(Uri uri, List? requestedSubProtocols) #endif _closeStatus = (WebSocketCloseStatus)code; _closeStatusDescription = reason; - _closeReceived = true; - WebSocketState state = State; - if (state == WebSocketState.Connecting || state == WebSocketState.Open || state == WebSocketState.CloseSent) - { - FastState = WebSocketState.Closed; - } #if FEATURE_WASM_THREADS } //lock #endif diff --git a/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs b/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs index 6f1e6faa490..1e99e356a87 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs +++ b/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Linq; using Xunit; using Xunit.Abstractions; @@ -264,8 +265,8 @@ public async Task CloseOutputAsync_ClientInitiated_CanReceive_CanClose(Uri serve [ActiveIssue("https://github.com/dotnet/runtime/issues/28957", typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] [OuterLoop("Uses external servers", typeof(PlatformDetection), nameof(PlatformDetection.LocalEchoServerIsNotAvailable))] - [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServers))] - public async Task CloseOutputAsync_ServerInitiated_CanReceive(Uri server) + [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServersWithSwitch))] + public async Task CloseOutputAsync_ServerInitiated_CanReceive(Uri server, bool delayReceiving) { string message = "Hello WebSockets!"; var expectedCloseStatus = WebSocketCloseStatus.NormalClosure; @@ -281,6 +282,10 @@ await cws.SendAsync( true, cts.Token); + // let server close the output before we request receiving + if (delayReceiving) + await Task.Delay(1000); + // Should be able to receive the message echoed by the server. var recvBuffer = new byte[100]; var segmentRecv = new ArraySegment(recvBuffer); @@ -367,6 +372,43 @@ await cws.SendAsync( } } + public static IEnumerable EchoServersWithSwitch => + EchoServers.SelectMany(server => new List + { + new object[] { server[0], true }, + new object[] { server[0], false } + }); + + [ActiveIssue("https://github.com/dotnet/runtime/issues/28957", typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] + [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServersWithSwitch))] + public async Task CloseOutputAsync_ServerInitiated_CanReceiveAfterClose(Uri server, bool syncState) + { + using (ClientWebSocket cws = await GetConnectedWebSocket(server, TimeOutMilliseconds, _output)) + { + var cts = new CancellationTokenSource(TimeOutMilliseconds); + await cws.SendAsync( + WebSocketData.GetBufferFromText(".receiveMessageAfterClose"), + WebSocketMessageType.Text, + true, + cts.Token); + + await Task.Delay(2000); + + if (syncState) + { + var state = cws.State; + Assert.Equal(WebSocketState.Open, state); + // should be able to receive after this sync + } + + var recvBuffer = new ArraySegment(new byte[1024]); + WebSocketReceiveResult recvResult = await cws.ReceiveAsync(recvBuffer, cts.Token); + var message = Encoding.UTF8.GetString(recvBuffer.ToArray(), 0, recvResult.Count); + + Assert.Contains(".receiveMessageAfterClose", message); + } + } + [OuterLoop("Uses external servers", typeof(PlatformDetection), nameof(PlatformDetection.LocalEchoServerIsNotAvailable))] [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServers))] public async Task CloseOutputAsync_CloseDescriptionIsNull_Success(Uri server) diff --git a/src/mono/wasm/runtime/exports-internal.ts b/src/mono/wasm/runtime/exports-internal.ts index 826180282e3..b009da17c6c 100644 --- a/src/mono/wasm/runtime/exports-internal.ts +++ b/src/mono/wasm/runtime/exports-internal.ts @@ -8,7 +8,7 @@ import { http_wasm_supports_streaming_response, http_wasm_create_abort_controler import { exportedRuntimeAPI, Module, runtimeHelpers } from "./globals"; import { get_property, set_property, has_property, get_typeof_property, get_global_this, dynamic_import } from "./invoke-js"; import { mono_wasm_stringify_as_error_with_stack } from "./logging"; -import { ws_wasm_create, ws_wasm_open, ws_wasm_send, ws_wasm_receive, ws_wasm_close, ws_wasm_abort } from "./web-socket"; +import { ws_wasm_create, ws_wasm_open, ws_wasm_send, ws_wasm_receive, ws_wasm_close, ws_wasm_abort, ws_get_state } from "./web-socket"; import { mono_wasm_get_loaded_files } from "./assets"; import { jiterpreter_dump_stats } from "./jiterpreter"; import { getOptions, applyOptions } from "./jiterpreter-support"; @@ -62,6 +62,7 @@ export function export_internal(): any { ws_wasm_receive, ws_wasm_close, ws_wasm_abort, + ws_get_state, // BrowserHttpHandler http_wasm_supports_streaming_response, diff --git a/src/mono/wasm/runtime/web-socket.ts b/src/mono/wasm/runtime/web-socket.ts index 5cf400aca53..0f99b7e2eaf 100644 --- a/src/mono/wasm/runtime/web-socket.ts +++ b/src/mono/wasm/runtime/web-socket.ts @@ -43,6 +43,17 @@ function verifyEnvironment() { } } +export function ws_get_state(ws: WebSocketExtension) : number +{ + if (ws.readyState != WebSocket.CLOSED) + return ws.readyState ?? -1; + const receive_event_queue = ws[wasm_ws_pending_receive_event_queue]; + const queued_events_count = receive_event_queue.getLength(); + if (queued_events_count == 0) + return ws.readyState ?? -1; + return WebSocket.OPEN; +} + export function ws_wasm_create(uri: string, sub_protocols: string[] | null, receive_status_ptr: VoidPtr, onClosed: (code: number, reason: string) => void): WebSocketExtension { verifyEnvironment(); mono_assert(uri && typeof uri === "string", () => `ERR12: Invalid uri ${typeof uri}`); @@ -175,8 +186,7 @@ export function ws_wasm_receive(ws: WebSocketExtension, buffer_ptr: VoidPtr, buf return null; } - const readyState = ws.readyState; - if (readyState == WebSocket.CLOSED) { + if (ws[wasm_ws_close_received]) { const receive_status_ptr = ws[wasm_ws_receive_status_ptr]; setI32(receive_status_ptr, 0); // count setI32(receive_status_ptr + 4, 2); // type:close From ba9df1e9210fd3e2d5631637eb9c70630ccc346c Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Sat, 30 Mar 2024 08:20:28 -0700 Subject: [PATCH 042/151] [release/8.0-staging] Call the Copy Constructor for stack arguments in C++/CLI on x86 (#100221) * Call the Copy Constructor for stack arguments in C++/CLI on Windows-x86 (#100050) * Add repro test case * Directly load the argument address using ldarga to avoid making a copy * Reimplement the "Copy Constructor Cookie" logic in a more modern and maintainable style to get the test passing again * Narrow support to Windows only --------- Co-authored-by: Jeremy Koritzinsky --- .../src/System/StubHelpers.cs | 69 ++++++++++++++ src/coreclr/vm/corelib.h | 15 ++++ src/coreclr/vm/dllimport.cpp | 56 ++++++++++++ src/coreclr/vm/dllimport.h | 7 ++ src/coreclr/vm/i386/asmhelpers.asm | 24 +++++ src/coreclr/vm/ilmarshalers.cpp | 36 ++++++++ src/coreclr/vm/metasig.h | 7 ++ .../CopyConstructorMarshaler.cs | 36 +++++++- .../IjwCopyConstructorMarshaler.cpp | 90 +++++++++++++++++++ .../Miscellaneous/CopyCtor/CopyCtorTest.cs | 28 +++++- 10 files changed, 361 insertions(+), 7 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs index 431c2f724a0..ced928ae2e7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs @@ -1111,6 +1111,75 @@ public IntPtr AddRef() } } // class CleanupWorkListElement + internal unsafe struct CopyConstructorCookie + { + private void* m_source; + + private nuint m_destinationOffset; + + public delegate* m_copyConstructor; + + public delegate* m_destructor; + + public CopyConstructorCookie* m_next; + + [StackTraceHidden] + public void ExecuteCopy(void* destinationBase) + { + if (m_copyConstructor != null) + { + m_copyConstructor((byte*)destinationBase + m_destinationOffset, m_source); + } + + if (m_destructor != null) + { + m_destructor(m_source); + } + } + } + + internal unsafe struct CopyConstructorChain + { + public void* m_realTarget; + public CopyConstructorCookie* m_head; + + public void Add(CopyConstructorCookie* cookie) + { + cookie->m_next = m_head; + m_head = cookie; + } + + [ThreadStatic] + private static CopyConstructorChain s_copyConstructorChain; + + public void Install(void* realTarget) + { + m_realTarget = realTarget; + s_copyConstructorChain = this; + } + + [StackTraceHidden] + private void ExecuteCopies(void* destinationBase) + { + for (CopyConstructorCookie* current = m_head; current != null; current = current->m_next) + { + current->ExecuteCopy(destinationBase); + } + } + + [UnmanagedCallersOnly] + [StackTraceHidden] + public static void* ExecuteCurrentCopiesAndGetTarget(void* destinationBase) + { + void* target = s_copyConstructorChain.m_realTarget; + s_copyConstructorChain.ExecuteCopies(destinationBase); + // Reset this instance to ensure we don't accidentally execute the copies again. + // All of the pointers point to the stack, so we don't need to free any memory. + s_copyConstructorChain = default; + return target; + } + } + internal static class StubHelpers { [MethodImpl(MethodImplOptions.InternalCall)] diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 96576a85813..cf059679bd0 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -1068,6 +1068,21 @@ DEFINE_METHOD(HANDLE_MARSHALER, CONVERT_SAFEHANDLE_TO_NATIVE,ConvertSaf DEFINE_METHOD(HANDLE_MARSHALER, THROW_SAFEHANDLE_FIELD_CHANGED, ThrowSafeHandleFieldChanged, SM_RetVoid) DEFINE_METHOD(HANDLE_MARSHALER, THROW_CRITICALHANDLE_FIELD_CHANGED, ThrowCriticalHandleFieldChanged, SM_RetVoid) +#ifdef TARGET_WINDOWS +#ifdef TARGET_X86 +DEFINE_CLASS(COPY_CONSTRUCTOR_CHAIN, StubHelpers, CopyConstructorChain) +DEFINE_METHOD(COPY_CONSTRUCTOR_CHAIN, EXECUTE_CURRENT_COPIES_AND_GET_TARGET, ExecuteCurrentCopiesAndGetTarget, SM_PtrVoid_RetPtrVoid) +DEFINE_METHOD(COPY_CONSTRUCTOR_CHAIN, INSTALL, Install, IM_PtrVoid_RetVoid) +DEFINE_METHOD(COPY_CONSTRUCTOR_CHAIN, ADD, Add, IM_PtrCopyConstructorCookie_RetVoid) + +DEFINE_CLASS(COPY_CONSTRUCTOR_COOKIE, StubHelpers, CopyConstructorCookie) +DEFINE_FIELD(COPY_CONSTRUCTOR_COOKIE, SOURCE, m_source) +DEFINE_FIELD(COPY_CONSTRUCTOR_COOKIE, DESTINATION_OFFSET, m_destinationOffset) +DEFINE_FIELD(COPY_CONSTRUCTOR_COOKIE, COPY_CONSTRUCTOR, m_copyConstructor) +DEFINE_FIELD(COPY_CONSTRUCTOR_COOKIE, DESTRUCTOR, m_destructor) +#endif // TARGET_X86 +#endif // TARGET_WINDOWS + DEFINE_CLASS(NATIVEVARIANT, StubHelpers, NativeVariant) DEFINE_CLASS(NATIVEDECIMAL, StubHelpers, NativeDecimal) diff --git a/src/coreclr/vm/dllimport.cpp b/src/coreclr/vm/dllimport.cpp index 37aab6a3f53..31c4467b798 100644 --- a/src/coreclr/vm/dllimport.cpp +++ b/src/coreclr/vm/dllimport.cpp @@ -1623,6 +1623,10 @@ NDirectStubLinker::NDirectStubLinker( m_pcsSetup->EmitSTLOC(m_dwTargetInterfacePointerLocalNum); } #endif // FEATURE_COMINTEROP + +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) + m_dwCopyCtorChainLocalNum = (DWORD)-1; +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) } void NDirectStubLinker::SetCallingConvention(CorInfoCallConvExtension unmngCallConv, BOOL fIsVarArg) @@ -1835,6 +1839,23 @@ DWORD NDirectStubLinker::GetReturnValueLocalNum() return m_dwRetValLocalNum; } +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) +DWORD NDirectStubLinker::GetCopyCtorChainLocalNum() +{ + STANDARD_VM_CONTRACT; + + if (m_dwCopyCtorChainLocalNum == (DWORD)-1) + { + // The local is created and initialized lazily when first asked. + m_dwCopyCtorChainLocalNum = NewLocal(CoreLibBinder::GetClass(CLASS__COPY_CONSTRUCTOR_CHAIN)); + m_pcsSetup->EmitLDLOCA(m_dwCopyCtorChainLocalNum); + m_pcsSetup->EmitINITOBJ(m_pcsSetup->GetToken(CoreLibBinder::GetClass(CLASS__COPY_CONSTRUCTOR_CHAIN))); + } + + return m_dwCopyCtorChainLocalNum; +} +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) + BOOL NDirectStubLinker::IsCleanupNeeded() { LIMITED_METHOD_CONTRACT; @@ -2064,6 +2085,10 @@ void NDirectStubLinker::End(DWORD dwStubFlags) } } +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) +EXTERN_C void STDCALL CopyConstructorCallStub(void); +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) + void NDirectStubLinker::DoNDirect(ILCodeStream *pcsEmit, DWORD dwStubFlags, MethodDesc * pStubMD) { STANDARD_VM_CONTRACT; @@ -2150,6 +2175,21 @@ void NDirectStubLinker::DoNDirect(ILCodeStream *pcsEmit, DWORD dwStubFlags, Meth } } +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) + if (m_dwCopyCtorChainLocalNum != (DWORD)-1) + { + // If we have a copy constructor chain local, we need to call the copy constructor stub + // to ensure that the chain is called correctly. + // Let's install the stub chain here and redirect the call to the stub. + DWORD targetLoc = NewLocal(ELEMENT_TYPE_I); + pcsEmit->EmitSTLOC(targetLoc); + pcsEmit->EmitLDLOCA(m_dwCopyCtorChainLocalNum); + pcsEmit->EmitLDLOC(targetLoc); + pcsEmit->EmitCALL(METHOD__COPY_CONSTRUCTOR_CHAIN__INSTALL, 2, 0); + pcsEmit->EmitLDC((DWORD_PTR)&CopyConstructorCallStub); + } +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) + // For managed-to-native calls, the rest of the work is done by the JIT. It will // erect InlinedCallFrame, flip GC mode, and use the specified calling convention // to call the target. For native-to-managed calls, this is an ordinary managed @@ -6078,5 +6118,21 @@ PCODE GetILStubForCalli(VASigCookie *pVASigCookie, MethodDesc *pMD) RETURN pVASigCookie->pNDirectILStub; } +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) +// Copy constructor support for C++/CLI +EXTERN_C void* STDCALL CallCopyConstructorsWorker(void* esp) +{ + STATIC_CONTRACT_THROWS; + STATIC_CONTRACT_GC_TRIGGERS; + STATIC_CONTRACT_MODE_PREEMPTIVE; // we've already switched to preemptive + + using ExecuteCallback = void*(STDMETHODCALLTYPE*)(void*); + + MethodDesc* pMD = CoreLibBinder::GetMethod(METHOD__COPY_CONSTRUCTOR_CHAIN__EXECUTE_CURRENT_COPIES_AND_GET_TARGET); + ExecuteCallback pExecute = (ExecuteCallback)pMD->GetMultiCallableAddrOfCode(); + + return pExecute(esp); +} +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) #endif // #ifndef DACCESS_COMPILE diff --git a/src/coreclr/vm/dllimport.h b/src/coreclr/vm/dllimport.h index 256b9507993..a89a01bc700 100644 --- a/src/coreclr/vm/dllimport.h +++ b/src/coreclr/vm/dllimport.h @@ -484,6 +484,9 @@ class NDirectStubLinker : public ILStubLinker DWORD GetCleanupWorkListLocalNum(); DWORD GetThreadLocalNum(); DWORD GetReturnValueLocalNum(); +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) + DWORD GetCopyCtorChainLocalNum(); +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) void SetCleanupNeeded(); void SetExceptionCleanupNeeded(); BOOL IsCleanupWorkListSetup(); @@ -553,6 +556,10 @@ class NDirectStubLinker : public ILStubLinker DWORD m_dwTargetEntryPointLocalNum; #endif // FEATURE_COMINTEROP +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) + DWORD m_dwCopyCtorChainLocalNum; +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) + BOOL m_fHasCleanupCode; BOOL m_fHasExceptionCleanupCode; BOOL m_fCleanupWorkListIsSetup; diff --git a/src/coreclr/vm/i386/asmhelpers.asm b/src/coreclr/vm/i386/asmhelpers.asm index e03ffa9544f..1d02fc48f8d 100644 --- a/src/coreclr/vm/i386/asmhelpers.asm +++ b/src/coreclr/vm/i386/asmhelpers.asm @@ -41,6 +41,7 @@ EXTERN _NDirectImportWorker@4:PROC EXTERN _VarargPInvokeStubWorker@12:PROC EXTERN _GenericPInvokeCalliStubWorker@12:PROC +EXTERN _CallCopyConstructorsWorker@4:PROC EXTERN _PreStubWorker@8:PROC EXTERN _TheUMEntryPrestubWorker@4:PROC @@ -1062,6 +1063,29 @@ GoCallCalliWorker: _GenericPInvokeCalliHelper@0 endp +;========================================================================== +; This is small stub whose purpose is to record current stack pointer and +; call CallCopyConstructorsWorker to invoke copy constructors and destructors +; as appropriate. This stub operates on arguments already pushed to the +; stack by JITted IL stub and must not create a new frame, i.e. it must tail +; call to the target for it to see the arguments that copy ctors have been +; called on. +; +_CopyConstructorCallStub@0 proc public + ; there may be an argument in ecx - save it + push ecx + + ; push pointer to arguments + lea edx, [esp + 8] + push edx + + call _CallCopyConstructorsWorker@4 + + ; restore ecx and tail call to the target + pop ecx + jmp eax +_CopyConstructorCallStub@0 endp + ifdef FEATURE_COMINTEROP ;========================================================================== diff --git a/src/coreclr/vm/ilmarshalers.cpp b/src/coreclr/vm/ilmarshalers.cpp index 431064f4e12..2e622bd725e 100644 --- a/src/coreclr/vm/ilmarshalers.cpp +++ b/src/coreclr/vm/ilmarshalers.cpp @@ -3459,6 +3459,38 @@ MarshalerOverrideStatus ILBlittableValueClassWithCopyCtorMarshaler::ArgumentOver #ifdef TARGET_X86 pslIL->SetStubTargetArgType(&locDesc); // native type is the value type pslILDispatch->EmitLDLOC(dwNewValueTypeLocal); // we load the local directly + +#if defined(TARGET_WINDOWS) + // Record this argument's stack slot in the copy constructor chain so we can correctly invoke the copy constructor. + DWORD ctorCookie = pslIL->NewLocal(CoreLibBinder::GetClass(CLASS__COPY_CONSTRUCTOR_COOKIE)); + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitINITOBJ(pslIL->GetToken(CoreLibBinder::GetClass(CLASS__COPY_CONSTRUCTOR_COOKIE))); + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitLDLOCA(dwNewValueTypeLocal); + pslIL->EmitSTFLD(pslIL->GetToken(CoreLibBinder::GetField(FIELD__COPY_CONSTRUCTOR_COOKIE__SOURCE))); + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitLDC(nativeStackOffset); + pslIL->EmitSTFLD(pslIL->GetToken(CoreLibBinder::GetField(FIELD__COPY_CONSTRUCTOR_COOKIE__DESTINATION_OFFSET))); + + if (pargs->mm.m_pCopyCtor) + { + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitLDFTN(pslIL->GetToken(pargs->mm.m_pCopyCtor)); + pslIL->EmitSTFLD(pslIL->GetToken(CoreLibBinder::GetField(FIELD__COPY_CONSTRUCTOR_COOKIE__COPY_CONSTRUCTOR))); + } + + if (pargs->mm.m_pDtor) + { + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitLDFTN(pslIL->GetToken(pargs->mm.m_pDtor)); + pslIL->EmitSTFLD(pslIL->GetToken(CoreLibBinder::GetField(FIELD__COPY_CONSTRUCTOR_COOKIE__DESTRUCTOR))); + } + + pslIL->EmitLDLOCA(psl->GetCopyCtorChainLocalNum()); + pslIL->EmitLDLOCA(ctorCookie); + pslIL->EmitCALL(METHOD__COPY_CONSTRUCTOR_CHAIN__ADD, 2, 0); +#endif // defined(TARGET_WINDOWS) + #else pslIL->SetStubTargetArgType(ELEMENT_TYPE_I); // native type is a pointer EmitLoadNativeLocalAddrForByRefDispatch(pslILDispatch, dwNewValueTypeLocal); @@ -3477,9 +3509,13 @@ MarshalerOverrideStatus ILBlittableValueClassWithCopyCtorMarshaler::ArgumentOver DWORD dwNewValueTypeLocal; dwNewValueTypeLocal = pslIL->NewLocal(locDesc); +#ifdef TARGET_WINDOWS + pslILDispatch->EmitLDARGA(argidx); +#else // !TARGET_WINDOWS pslILDispatch->EmitLDARG(argidx); pslILDispatch->EmitSTLOC(dwNewValueTypeLocal); pslILDispatch->EmitLDLOCA(dwNewValueTypeLocal); +#endif // TARGET_WINDOWS #else LocalDesc locDesc(pargs->mm.m_pMT); locDesc.MakeCopyConstructedPointer(); diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h index 6adf044a017..ed277458c9b 100644 --- a/src/coreclr/vm/metasig.h +++ b/src/coreclr/vm/metasig.h @@ -578,6 +578,13 @@ DEFINE_METASIG_T(SM(RefCleanupWorkListElement_RetVoid, r(C(CLEANUP_WORK_LIST_ELE DEFINE_METASIG_T(SM(RefCleanupWorkListElement_SafeHandle_RetIntPtr, r(C(CLEANUP_WORK_LIST_ELEMENT)) C(SAFE_HANDLE), I)) DEFINE_METASIG_T(SM(RefCleanupWorkListElement_Obj_RetVoid, r(C(CLEANUP_WORK_LIST_ELEMENT)) j, v)) +DEFINE_METASIG(SM(PtrVoid_RetPtrVoid, P(v), P(v))) +DEFINE_METASIG(IM(PtrVoid_RetVoid, P(v), v)) +#if defined(TARGET_X86) && defined(TARGET_WINDOWS) +DEFINE_METASIG_T(IM(PtrCopyConstructorCookie_RetVoid, P(g(COPY_CONSTRUCTOR_COOKIE)), v)) +#endif // defined(TARGET_X86) && defined(TARGET_WINDOWS) + + #ifdef FEATURE_ICASTABLE DEFINE_METASIG_T(SM(ICastable_RtType_RefException_RetBool, C(ICASTABLE) C(CLASS) r(C(EXCEPTION)), F)) DEFINE_METASIG_T(SM(ICastable_RtType_RetRtType, C(ICASTABLE) C(CLASS), C(CLASS))) diff --git a/src/tests/Interop/IJW/CopyConstructorMarshaler/CopyConstructorMarshaler.cs b/src/tests/Interop/IJW/CopyConstructorMarshaler/CopyConstructorMarshaler.cs index 95afea492be..baf3ebd4579 100644 --- a/src/tests/Interop/IJW/CopyConstructorMarshaler/CopyConstructorMarshaler.cs +++ b/src/tests/Interop/IJW/CopyConstructorMarshaler/CopyConstructorMarshaler.cs @@ -25,34 +25,62 @@ static int Main() object testInstance = Activator.CreateInstance(testType); MethodInfo testMethod = testType.GetMethod("PInvokeNumCopies"); + // On x86, we have an additional copy on every P/Invoke from the "native" parameter to the actual location on the stack. + int platformExtra = 0; + if (RuntimeInformation.ProcessArchitecture == Architecture.X86) + { + platformExtra = 1; + } + // PInvoke will copy twice. Once from argument to parameter, and once from the managed to native parameter. - Assert.Equal(2, (int)testMethod.Invoke(testInstance, null)); + Assert.Equal(2 + platformExtra, (int)testMethod.Invoke(testInstance, null)); testMethod = testType.GetMethod("ReversePInvokeNumCopies"); // Reverse PInvoke will copy 3 times. Two are from the same paths as the PInvoke, // and the third is from the reverse P/Invoke call. - Assert.Equal(3, (int)testMethod.Invoke(testInstance, null)); + Assert.Equal(3 + platformExtra, (int)testMethod.Invoke(testInstance, null)); testMethod = testType.GetMethod("PInvokeNumCopiesDerivedType"); // PInvoke will copy twice. Once from argument to parameter, and once from the managed to native parameter. - Assert.Equal(2, (int)testMethod.Invoke(testInstance, null)); + Assert.Equal(2 + platformExtra, (int)testMethod.Invoke(testInstance, null)); testMethod = testType.GetMethod("ReversePInvokeNumCopiesDerivedType"); // Reverse PInvoke will copy 3 times. Two are from the same paths as the PInvoke, // and the third is from the reverse P/Invoke call. - Assert.Equal(3, (int)testMethod.Invoke(testInstance, null)); + Assert.Equal(3 + platformExtra, (int)testMethod.Invoke(testInstance, null)); } catch (Exception ex) { Console.WriteLine(ex); return 101; } + + try + { + CopyConstructorsInArgumentStackSlots(); + } + catch (Exception ex) + { + Console.WriteLine(ex); + return 101; + } + return 100; } + public static void CopyConstructorsInArgumentStackSlots() + { + Assembly ijwNativeDll = Assembly.Load("IjwCopyConstructorMarshaler"); + Type testType = ijwNativeDll.GetType("TestClass"); + object testInstance = Activator.CreateInstance(testType); + MethodInfo testMethod = testType.GetMethod("ExposedThisCopyConstructorScenario"); + + Assert.Equal(0, (int)testMethod.Invoke(testInstance, null)); + } + [DllImport("kernel32.dll")] static extern IntPtr LoadLibraryEx(string lpFileName, IntPtr hReservedNull, int dwFlags); diff --git a/src/tests/Interop/IJW/CopyConstructorMarshaler/IjwCopyConstructorMarshaler.cpp b/src/tests/Interop/IJW/CopyConstructorMarshaler/IjwCopyConstructorMarshaler.cpp index bd1d1b80829..c3d50cf7783 100644 --- a/src/tests/Interop/IJW/CopyConstructorMarshaler/IjwCopyConstructorMarshaler.cpp +++ b/src/tests/Interop/IJW/CopyConstructorMarshaler/IjwCopyConstructorMarshaler.cpp @@ -1,5 +1,90 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma unmanaged +#include +#include + +namespace ExposedThis +{ + struct Relative; + + std::vector relatives; + + int numMissedCopies = 0; + + struct Relative + { + void* relative; + Relative() + { + std::cout << "Registering " << std::hex << this << "\n"; + relatives.push_back(this); + relative = this - 1; + } + + Relative(const Relative& other) + { + std::cout << "Registering copy of " << std::hex << &other << " at " << this << "\n"; + relatives.push_back(this); + relative = this - 1; + } + + ~Relative() + { + auto location = std::find(relatives.begin(), relatives.end(), this); + if (location != relatives.end()) + { + std::cout << "Unregistering " << std::hex << this << "\n"; + relatives.erase(location); + } + else + { + std::cout << "Error: Relative object " << std::hex << this << " not registered\n"; + numMissedCopies++; + } + + if (relative != this - 1) + { + std::cout << " Error: Relative object " << std::hex << this << " has invalid relative pointer " << std::hex << relative << "\n"; + numMissedCopies++; + } + } + }; + + void UseRelative(Relative rel) + { + std::cout << "Unmanaged: Using relative at address " << std::hex << &rel << "\n"; + } + + void UseRelativeManaged(Relative rel); + + void CallRelative() + { + Relative rel; + UseRelativeManaged(rel); + } + +#pragma managed + + int RunScenario() + { + // Managed to unmanaged + { + Relative rel; + UseRelative(rel); + } + + // Unmanaged to managed + CallRelative(); + + return numMissedCopies; + } + + void UseRelativeManaged(Relative rel) + { + std::cout << "Managed: Using relative at address " << std::hex << &rel << "\n"; + } +} #pragma managed class A @@ -102,4 +187,9 @@ public ref class TestClass B b; return GetCopyCount_ViaManaged(b); } + + int ExposedThisCopyConstructorScenario() + { + return ExposedThis::RunScenario(); + } }; diff --git a/src/tests/Interop/PInvoke/Miscellaneous/CopyCtor/CopyCtorTest.cs b/src/tests/Interop/PInvoke/Miscellaneous/CopyCtor/CopyCtorTest.cs index 37e8e3c48f5..ace0f2ba315 100644 --- a/src/tests/Interop/PInvoke/Miscellaneous/CopyCtor/CopyCtorTest.cs +++ b/src/tests/Interop/PInvoke/Miscellaneous/CopyCtor/CopyCtorTest.cs @@ -13,21 +13,43 @@ static unsafe class CopyCtor public static unsafe int StructWithCtorTest(StructWithCtor* ptrStruct, ref StructWithCtor refStruct) { if (ptrStruct->_instanceField != 1) + { + Console.WriteLine($"Fail: {ptrStruct->_instanceField} != {1}"); return 1; + } if (refStruct._instanceField != 2) + { + Console.WriteLine($"Fail: {refStruct._instanceField} != {2}"); return 2; + } - if (StructWithCtor.CopyCtorCallCount != 2) + int expectedCallCount = 2; + if (RuntimeInformation.ProcessArchitecture == Architecture.X86) + { + expectedCallCount = 4; + } + + if (StructWithCtor.CopyCtorCallCount != expectedCallCount) + { + Console.WriteLine($"Fail: {StructWithCtor.CopyCtorCallCount} != {expectedCallCount}"); return 3; - if (StructWithCtor.DtorCallCount != 2) + } + if (StructWithCtor.DtorCallCount != expectedCallCount) + { + Console.WriteLine($"Fail: {StructWithCtor.DtorCallCount} != {expectedCallCount}"); return 4; - + } return 100; } public static unsafe int Main() { + if (!TestLibrary.PlatformDetection.IsWindows) + { + return 100; + } + TestDelegate del = (TestDelegate)Delegate.CreateDelegate(typeof(TestDelegate), typeof(CopyCtor).GetMethod("StructWithCtorTest")); StructWithCtor s1 = new StructWithCtor(); StructWithCtor s2 = new StructWithCtor(); From 3ea18f2b16a8ead9d888e26d031f264abe27b12c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 11:34:12 +0200 Subject: [PATCH 043/151] [release/8.0] Don't unset ALPN list pointer during ALPN selection callback. (#99670) * Don't unset ALPN list pointer during ALPN selection callback. * Move AlpnHandle.Free to ReleaseHandle --------- Co-authored-by: Radek Zikmund --- .../Interop.OpenSsl.cs | 3 --- .../Interop.Ssl.cs | 12 ++++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs index d180aa98538..c526f37b0b9 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs @@ -694,9 +694,6 @@ private static unsafe int AlpnServerSelectCallback(IntPtr ssl, byte** outp, byte return Ssl.SSL_TLSEXT_ERR_ALERT_FATAL; } - // reset application data to avoid dangling pointer. - Ssl.SslSetData(ssl, IntPtr.Zero); - GCHandle protocolHandle = GCHandle.FromIntPtr(sslData); if (!(protocolHandle.Target is List protocolList)) { diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs index 71d23c26d34..0848ca5ed21 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.Ssl.cs @@ -420,12 +420,6 @@ protected override void Dispose(bool disposing) _writeBio?.Dispose(); } - if (AlpnHandle.IsAllocated) - { - Interop.Ssl.SslSetData(handle, IntPtr.Zero); - AlpnHandle.Free(); - } - base.Dispose(disposing); } @@ -438,6 +432,12 @@ protected override bool ReleaseHandle() SslContextHandle?.DangerousRelease(); + if (AlpnHandle.IsAllocated) + { + Interop.Ssl.SslSetData(handle, IntPtr.Zero); + AlpnHandle.Free(); + } + IntPtr h = handle; SetHandle(IntPtr.Zero); Interop.Ssl.SslDestroy(h); // will free the handles underlying _readBio and _writeBio From 9de64a25bb81552de32a928c4219b09ffa49e30d Mon Sep 17 00:00:00 2001 From: Thays Grazia Date: Tue, 2 Apr 2024 15:34:38 -0300 Subject: [PATCH 044/151] Backport of #99843 (#99844) --- src/mono/mono/component/debugger-agent.c | 1 + .../debugger/DebuggerTestSuite/MiscTests.cs | 19 +++++++++++++++++++ .../tests/debugger-test/debugger-test.cs | 9 +++++++++ 3 files changed, 29 insertions(+) diff --git a/src/mono/mono/component/debugger-agent.c b/src/mono/mono/component/debugger-agent.c index 80918c38876..5389f30803c 100644 --- a/src/mono/mono/component/debugger-agent.c +++ b/src/mono/mono/component/debugger-agent.c @@ -5589,6 +5589,7 @@ decode_value_compute_size (MonoType *t, int type, MonoDomain *domain, guint8 *bu if (type != t->type && !MONO_TYPE_IS_REFERENCE (t) && !(t->type == MONO_TYPE_I && type == MONO_TYPE_VALUETYPE) && !(type == VALUE_TYPE_ID_FIXED_ARRAY) && + !(type == MDBGPROT_VALUE_TYPE_ID_NULL) && !(t->type == MONO_TYPE_U && type == MONO_TYPE_VALUETYPE) && !(t->type == MONO_TYPE_PTR && type == MONO_TYPE_I8) && !(t->type == MONO_TYPE_FNPTR && type == MONO_TYPE_I8) && diff --git a/src/mono/wasm/debugger/DebuggerTestSuite/MiscTests.cs b/src/mono/wasm/debugger/DebuggerTestSuite/MiscTests.cs index 35fd867cb7a..5a65521a137 100644 --- a/src/mono/wasm/debugger/DebuggerTestSuite/MiscTests.cs +++ b/src/mono/wasm/debugger/DebuggerTestSuite/MiscTests.cs @@ -1188,5 +1188,24 @@ public async Task TestDebugUsingMultiThreadedRuntime() Assert.Equal(locals[1]["value"]["type"], "number"); Assert.Equal(locals[1]["name"], "currentThread"); } + + [Fact] + public async Task InspectSpanByte() + { + var expression = $"{{ invoke_static_method('[debugger-test] SpanByte:Run'); }}"; + + await EvaluateAndCheck( + "window.setTimeout(function() {" + expression + "; }, 1);", + "dotnet://debugger-test.dll/debugger-test.cs", 1664, 8, + "SpanByte.Run", + wait_for_event_fn: async (pause_location) => + { + var id = pause_location["callFrames"][0]["callFrameId"].Value(); + await EvaluateOnCallFrameAndCheck(id, + ("span", TObject("System.Span", null)) + ); + } + ); + } } } diff --git a/src/mono/wasm/debugger/tests/debugger-test/debugger-test.cs b/src/mono/wasm/debugger/tests/debugger-test/debugger-test.cs index 4a43032a4b3..b69baf08552 100644 --- a/src/mono/wasm/debugger/tests/debugger-test/debugger-test.cs +++ b/src/mono/wasm/debugger/tests/debugger-test/debugger-test.cs @@ -1656,3 +1656,12 @@ public void CallMethod() public int myField; } + +public class SpanByte +{ + public static void Run() + { + System.Span span = new (); + System.Diagnostics.Debugger.Break(); + } +} \ No newline at end of file From f8132e673bfba45e71db14eac12cf1effddbac41 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:25:58 -0700 Subject: [PATCH 045/151] Update branding to 8.0.5 (#100547) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 5fb45746a0f..a43374e3220 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 8.0.4 + 8.0.5 8 0 - 4 + 5 8.0.100 7.0.$([MSBuild]::Add($(PatchVersion),14)) 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) From 7d45915a7e7ba7b830d90998eca4cd064b6626b7 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Thu, 4 Apr 2024 13:37:06 -0700 Subject: [PATCH 046/151] [release/8.0-staging] Ensure that Sse3.MoveAndDuplicate correctly tracks supporting SIMD scalar loads (#100417) * Ensure that Sse3.MoveAndDuplicate correctly tracks supporting SIMD scalar loads (#97783) * Add a regression test for dotnet/runtime#100404 * Fix the regression test to not be called Main --- src/coreclr/jit/lowerxarch.cpp | 37 ++++++++++++++----- .../JitBlue/Runtime_100404/Runtime_100404.cs | 31 ++++++++++++++++ .../Runtime_100404/Runtime_100404.csproj | 8 ++++ 3 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.csproj diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f0e7fc6322f..dad1df9ffcf 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -7652,15 +7652,17 @@ bool Lowering::IsContainableHWIntrinsicOp(GenTreeHWIntrinsic* parentNode, GenTre } case NI_SSE2_ConvertToVector128Double: - case NI_SSE3_MoveAndDuplicate: case NI_AVX_ConvertToVector256Double: + case NI_AVX512F_ConvertToVector512Double: + case NI_AVX512F_VL_ConvertToVector128Double: + case NI_AVX512F_VL_ConvertToVector256Double: { assert(!supportsSIMDScalarLoads); // Most instructions under the non-VEX encoding require aligned operands. // Those used for Sse2.ConvertToVector128Double (CVTDQ2PD and CVTPS2PD) - // and Sse3.MoveAndDuplicate (MOVDDUP) are exceptions and don't fail for - // unaligned inputs as they read mem64 (half the vector width) instead + // are exceptions and don't fail for unaligned inputs as they read half + // the vector width instead supportsAlignedSIMDLoads = !comp->opts.MinOpts(); supportsUnalignedSIMDLoads = true; @@ -7668,10 +7670,29 @@ bool Lowering::IsContainableHWIntrinsicOp(GenTreeHWIntrinsic* parentNode, GenTre const unsigned expectedSize = genTypeSize(parentNode->TypeGet()) / 2; const unsigned operandSize = genTypeSize(childNode->TypeGet()); - // For broadcasts we can only optimize constants and memory operands - const bool broadcastIsContainable = childNode->OperIsConst() || childNode->isMemoryOp(); - supportsGeneralLoads = - broadcastIsContainable && supportsUnalignedSIMDLoads && (operandSize >= expectedSize); + if (childNode->OperIsConst() || childNode->isMemoryOp()) + { + // For broadcasts we can only optimize constants and memory operands + // since we're going from a smaller base type to a larger base type + supportsGeneralLoads = supportsUnalignedSIMDLoads && (operandSize >= expectedSize); + } + break; + } + + case NI_SSE3_MoveAndDuplicate: + { + // Most instructions under the non-VEX encoding require aligned operands. + // Those used for Sse3.MoveAndDuplicate (MOVDDUP) are exceptions and don't + // fail for unaligned inputs as they read half the vector width instead + + supportsAlignedSIMDLoads = !comp->opts.MinOpts(); + supportsUnalignedSIMDLoads = true; + + const unsigned expectedSize = genTypeSize(parentNode->TypeGet()) / 2; + const unsigned operandSize = genTypeSize(childNode->TypeGet()); + + supportsGeneralLoads = supportsUnalignedSIMDLoads && (operandSize >= expectedSize); + supportsSIMDScalarLoads = true; break; } @@ -7697,8 +7718,6 @@ bool Lowering::IsContainableHWIntrinsicOp(GenTreeHWIntrinsic* parentNode, GenTre break; } } - - assert(supportsSIMDScalarLoads == false); break; } diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.cs b/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.cs new file mode 100644 index 00000000000..0725a973fae --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; +using Xunit; + +public static class Runtime_100404 +{ + [Fact] + [MethodImpl(MethodImplOptions.NoInlining)] + public static void TestMultiplyVector128DoubleByConstant() + { + Vector128 result = Map(Vector128.One, new FloatPoint(2.0, 3.0)); + Assert.Equal(2.0, result[0]); + Assert.Equal(2.0, result[1]); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static Vector128 Map(Vector128 m0, FloatPoint point) + { + return m0 * Vector128.Create(point.X); + } + + private struct FloatPoint(double x, double y) + { + public double X = x; + public double Y = y; + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.csproj new file mode 100644 index 00000000000..15edd99711a --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100404/Runtime_100404.csproj @@ -0,0 +1,8 @@ + + + True + + + + + \ No newline at end of file From 7ae2d0071e394fcafeec4f55e9b5faff5992b88e Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Thu, 4 Apr 2024 14:40:03 -0700 Subject: [PATCH 047/151] Fix allocation of empty array in the frozen heap for collectible types (#100444) (#100509) * Fix allocation of empty array in the frozen heap for collectible types (#100437) * Remove Optimize from csproj * Add test for generic with static * Apply suggestions from code review * Better test * Disable tests on Mono --------- Co-authored-by: Alexandre Mutel --- src/coreclr/jit/importer.cpp | 3 +- src/coreclr/vm/frozenobjectheap.cpp | 1 + .../JitBlue/Runtime_100437/Runtime_100437.cs | 112 ++++++++++++++++++ .../Runtime_100437/Runtime_100437.csproj | 5 + 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.csproj diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 6087b149930..9bf654ffc35 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -9393,7 +9393,8 @@ void Compiler::impImportBlockCode(BasicBlock* block) CORINFO_FIELD_INFO fi; eeGetFieldInfo(&fldToken, CORINFO_ACCESS_SET, &fi); unsigned flagsToCheck = CORINFO_FLG_FIELD_STATIC | CORINFO_FLG_FIELD_FINAL; - if ((fi.fieldFlags & flagsToCheck) == flagsToCheck) + if (((fi.fieldFlags & flagsToCheck) == flagsToCheck) && + ((info.compCompHnd->getClassAttribs(info.compClassHnd) & CORINFO_FLG_SHAREDINST) == 0)) { #ifdef FEATURE_READYTORUN if (opts.IsReadyToRun()) diff --git a/src/coreclr/vm/frozenobjectheap.cpp b/src/coreclr/vm/frozenobjectheap.cpp index 8f11f3c8c74..bd25b4ba7b3 100644 --- a/src/coreclr/vm/frozenobjectheap.cpp +++ b/src/coreclr/vm/frozenobjectheap.cpp @@ -47,6 +47,7 @@ Object* FrozenObjectHeapManager::TryAllocateObject(PTR_MethodTable type, size_t _ASSERT(type != nullptr); _ASSERT(FOH_COMMIT_SIZE >= MIN_OBJECT_SIZE); + _ASSERT(!type->Collectible()); // Currently we don't support frozen objects with special alignment requirements // TODO: We should also give up on arrays of doubles on 32-bit platforms. diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.cs b/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.cs new file mode 100644 index 00000000000..810b99acab3 --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.Loader; +using Xunit; + +public class Runtime_100437 +{ + [Fact] + [SkipOnMono("PlatformDetection.IsPreciseGcSupported false on mono", TestPlatforms.Any)] + public static void TestNonCollectibleType() => TestCollectibleReadOnlyStatics(nameof(NonCollectibleType)); + + [Fact] + [SkipOnMono("PlatformDetection.IsPreciseGcSupported false on mono", TestPlatforms.Any)] + public static void TestNonCollectibleTypeInSharedGenericCode() => TestCollectibleReadOnlyStatics(nameof(NonCollectibleTypeInSharedGenericCode)); + + [Fact] + [SkipOnMono("PlatformDetection.IsPreciseGcSupported false on mono", TestPlatforms.Any)] + public static void TestNonCollectibleArrayTypeInSharedGenericCode() => TestCollectibleReadOnlyStatics(nameof(NonCollectibleArrayTypeInSharedGenericCode)); + + [Fact] + [SkipOnMono("PlatformDetection.IsPreciseGcSupported false on mono", TestPlatforms.Any)] + public static void TestCollectibleEmptyArray() => TestCollectibleReadOnlyStatics(nameof(CollectibleEmptyArray)); + + private static void TestCollectibleReadOnlyStatics(string methodName) + { + string assemblyPath = typeof(Runtime_100437).Assembly.Location; + + // Skip this test for single file + if (string.IsNullOrEmpty(assemblyPath)) + return; + + WeakReference wr = CreateReadOnlyStaticWeakReference(); + + for (int i = 0; i < 10; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + + if (!IsTargetAlive(wr)) + return; + } + + throw new Exception("Test failed - readonly static has not been collected."); + + [MethodImpl(MethodImplOptions.NoInlining)] + WeakReference CreateReadOnlyStaticWeakReference() + { + AssemblyLoadContext alc = new CollectibleAssemblyLoadContext(); + Assembly a = alc.LoadFromAssemblyPath(assemblyPath); + return (WeakReference)a.GetType(nameof(Runtime_100437)).GetMethod(methodName).Invoke(null, new object[] { typeof(Runtime_100437).Assembly }); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + bool IsTargetAlive(WeakReference wr) + { + return wr.Target != null; + } + } + + public static WeakReference NonCollectibleType(Assembly assemblyInDefaultContext) + { + return new WeakReference(Holder.Singleton, trackResurrection: true); + } + + public static WeakReference NonCollectibleTypeInSharedGenericCode(Assembly assemblyInDefaultContext) + { + // Create instance of a non-collectible generic type definition over a collectible type + var type = assemblyInDefaultContext.GetType("Runtime_100437+GenericHolder`1", throwOnError: true).MakeGenericType(typeof(Runtime_100437)); + var field = type.GetField("Singleton", BindingFlags.Static | BindingFlags.Public); + return new WeakReference(field.GetValue(null), trackResurrection: true); + } + + public static WeakReference NonCollectibleArrayTypeInSharedGenericCode(Assembly assemblyInDefaultContext) + { + // Create instance of a non-collectible generic type definition over a collectible type + var type = assemblyInDefaultContext.GetType("Runtime_100437+GenericArrayHolder`1", throwOnError: true).MakeGenericType(typeof(Runtime_100437)); + var field = type.GetField("Singleton", BindingFlags.Static | BindingFlags.Public); + return new WeakReference(field.GetValue(null), trackResurrection: true); + } + + public static WeakReference CollectibleEmptyArray(Assembly assemblyInDefaultContext) + { + return new WeakReference(Array.Empty(), trackResurrection: true); + } + + private class CollectibleAssemblyLoadContext : AssemblyLoadContext + { + public CollectibleAssemblyLoadContext() + : base(isCollectible: true) + { + } + } + + private class Holder + { + public static readonly object Singleton = new object(); + } + + private class GenericHolder + { + public static readonly object Singleton = new object(); + } + + private class GenericArrayHolder + { + public static readonly int[] Singleton = new int[0]; + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.csproj new file mode 100644 index 00000000000..197767e2c4e --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100437/Runtime_100437.csproj @@ -0,0 +1,5 @@ + + + + + From 479defb07667b30c3eb69bdcea00e75e1ef0b015 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:42:15 -0700 Subject: [PATCH 048/151] Use clang --version to get Apple toolset version (#100199) Fixes #100189 Co-authored-by: Jan Kotas --- .../BuildIntegration/Microsoft.NETCore.Native.Unix.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index c9ae3083cf5..8013121de4f 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -166,7 +166,7 @@ The .NET Foundation licenses this file to you under the MIT license. - + From d49f6cfd43af1e022d96d1a62c441134aff1f813 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Apr 2024 13:26:58 -0700 Subject: [PATCH 049/151] Azure Linux 3.0 deps package (#100684) Co-authored-by: Nikola Milosavljevic --- src/installer/pkg/sfx/installers.proj | 1 + .../dotnet-runtime-deps/dotnet-runtime-deps-azl.3.proj | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 src/installer/pkg/sfx/installers/dotnet-runtime-deps/dotnet-runtime-deps-azl.3.proj diff --git a/src/installer/pkg/sfx/installers.proj b/src/installer/pkg/sfx/installers.proj index 06e366db911..7f4ce6b9c1c 100644 --- a/src/installer/pkg/sfx/installers.proj +++ b/src/installer/pkg/sfx/installers.proj @@ -10,6 +10,7 @@ + diff --git a/src/installer/pkg/sfx/installers/dotnet-runtime-deps/dotnet-runtime-deps-azl.3.proj b/src/installer/pkg/sfx/installers/dotnet-runtime-deps/dotnet-runtime-deps-azl.3.proj new file mode 100644 index 00000000000..ee363967c36 --- /dev/null +++ b/src/installer/pkg/sfx/installers/dotnet-runtime-deps/dotnet-runtime-deps-azl.3.proj @@ -0,0 +1,10 @@ + + + false + azl.3 + + + + + + From 2b94204aa04ae15806c559820517cba31acf1b2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:31:12 +0200 Subject: [PATCH 050/151] Fix Http2 deadlock (#100086) Co-authored-by: Miha Zupan --- .../SocketsHttpHandler/Http2Connection.cs | 23 +++++++++++++++---- .../SocketsHttpHandler/HttpConnectionPool.cs | 6 ++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index 5b795f0a884..54857380a3a 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -257,6 +257,7 @@ private void Shutdown() if (NetEventSource.Log.IsEnabled()) Trace($"{nameof(_shutdown)}={_shutdown}, {nameof(_abortException)}={_abortException}"); Debug.Assert(Monitor.IsEntered(SyncObject)); + Debug.Assert(!_pool.HasSyncObjLock); if (!_shutdown) { @@ -276,6 +277,8 @@ private void Shutdown() public bool TryReserveStream() { + Debug.Assert(!_pool.HasSyncObjLock); + lock (SyncObject) { if (_shutdown) @@ -302,6 +305,8 @@ public bool TryReserveStream() // Otherwise, will be called when the request is complete and stream is closed. public void ReleaseStream() { + Debug.Assert(!_pool.HasSyncObjLock); + lock (SyncObject) { if (NetEventSource.Log.IsEnabled()) Trace($"{nameof(_streamsInUse)}={_streamsInUse}"); @@ -333,6 +338,8 @@ public void ReleaseStream() // Returns false to indicate that the connection is shutting down and cannot be used anymore public Task WaitForAvailableStreamsAsync() { + Debug.Assert(!_pool.HasSyncObjLock); + lock (SyncObject) { Debug.Assert(_availableStreamsWaiter is null, "As used currently, shouldn't already have a waiter"); @@ -730,6 +737,7 @@ private void ProcessAltSvcFrame(FrameHeader frameHeader) { if (NetEventSource.Log.IsEnabled()) Trace($"{frameHeader}"); Debug.Assert(frameHeader.Type == FrameType.AltSvc); + Debug.Assert(!Monitor.IsEntered(SyncObject)); ReadOnlySpan span = _incomingBuffer.ActiveSpan.Slice(0, frameHeader.PayloadLength); @@ -1308,6 +1316,8 @@ private Task SendRstStreamAsync(int streamId, Http2ProtocolErrorCode errorCode) internal void HeartBeat() { + Debug.Assert(!_pool.HasSyncObjLock); + if (_shutdown) return; @@ -1800,10 +1810,14 @@ private void ExtendWindow(int amount) public override long GetIdleTicks(long nowTicks) { - lock (SyncObject) - { - return _streamsInUse == 0 ? base.GetIdleTicks(nowTicks) : 0; - } + // The pool is holding the lock as part of its scavenging logic. + // We must not lock on Http2Connection.SyncObj here as that could lead to lock ordering problems. + Debug.Assert(_pool.HasSyncObjLock); + + // There is a race condition here where the connection pool may see this connection as idle right before + // we start processing a new request and start its disposal. This is okay as we will either + // return false from TryReserveStream, or process pending requests before tearing down the transport. + return _streamsInUse == 0 ? base.GetIdleTicks(nowTicks) : 0; } /// Abort all streams and cause further processing to fail. @@ -1992,6 +2006,7 @@ private static TaskCompletionSourceWithCancellation CreateSuccessfullyComp public async Task SendAsync(HttpRequestMessage request, bool async, CancellationToken cancellationToken) { Debug.Assert(async); + Debug.Assert(!_pool.HasSyncObjLock); if (NetEventSource.Log.IsEnabled()) Trace($"Sending request: {request}"); try diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs index 163a9492ab4..17ed807eeef 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs @@ -406,7 +406,7 @@ private object SyncObj } } - private bool HasSyncObjLock => Monitor.IsEntered(_availableHttp11Connections); + public bool HasSyncObjLock => Monitor.IsEntered(_availableHttp11Connections); // Overview of connection management (mostly HTTP version independent): // @@ -459,6 +459,8 @@ private static void ThrowGetVersionException(HttpRequestMessage request, int des private bool CheckExpirationOnGet(HttpConnectionBase connection) { + Debug.Assert(!HasSyncObjLock); + TimeSpan pooledConnectionLifetime = _poolManager.Settings._pooledConnectionLifetime; if (pooledConnectionLifetime != Timeout.InfiniteTimeSpan) { @@ -2000,6 +2002,7 @@ private void ReturnHttp2Connection(Http2Connection connection, bool isNewConnect { if (NetEventSource.Log.IsEnabled()) connection.Trace($"{nameof(isNewConnection)}={isNewConnection}"); + Debug.Assert(!HasSyncObjLock); Debug.Assert(isNewConnection || initialRequestWaiter is null, "Shouldn't have a request unless the connection is new"); if (!isNewConnection && CheckExpirationOnReturn(connection)) @@ -2403,6 +2406,7 @@ internal void HeartBeat() localHttp2Connections = _availableHttp2Connections?.ToArray(); } + // Avoid calling HeartBeat under the lock, as it may call back into HttpConnectionPool.InvalidateHttp2Connection. if (localHttp2Connections is not null) { foreach (Http2Connection http2Connection in localHttp2Connections) From c031937d06e535f22785cd13198b47c3f72adbc1 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Tue, 9 Apr 2024 13:12:37 -0500 Subject: [PATCH 051/151] Fix unbound MAC work in GetCertContentType --- .../X509Certificates/X509Pal.macOS.cs | 7 +++- ...PfxIterationCountTests.X509Certificate2.cs | 29 +++++++++++++ .../tests/X509Certificates/TestData.cs | 20 +++++++++ .../pal_x509.c | 41 +++---------------- 4 files changed, 59 insertions(+), 38 deletions(-) diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X509Pal.macOS.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X509Pal.macOS.cs index 7a6e555b12c..a9cda7f9735 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X509Pal.macOS.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X509Pal.macOS.cs @@ -111,6 +111,7 @@ private static DSA DecodeDsaPublicKey(byte[] encodedKeyValue, byte[] encodedPara public X509ContentType GetCertContentType(ReadOnlySpan rawData) { const int errSecUnknownFormat = -25257; + if (rawData.IsEmpty) { // Throw to match Windows and Unix behavior. @@ -119,7 +120,7 @@ public X509ContentType GetCertContentType(ReadOnlySpan rawData) X509ContentType contentType = Interop.AppleCrypto.X509GetContentType(rawData); - // Apple doesn't seem to recognize PFX files with no MAC, so try a quick maybe-it's-a-PFX test + // Apple's native check can't check for PKCS12, so do a quick decode test to see if it is PKCS12 / PFX. if (contentType == X509ContentType.Unknown) { try @@ -128,9 +129,11 @@ public X509ContentType GetCertContentType(ReadOnlySpan rawData) { fixed (byte* pin = rawData) { + AsnValueReader reader = new AsnValueReader(rawData, AsnEncodingRules.BER); + using (var manager = new PointerMemoryManager(pin, rawData.Length)) { - PfxAsn.Decode(manager.Memory, AsnEncodingRules.BER); + PfxAsn.Decode(ref reader, manager.Memory, out _); } contentType = X509ContentType.Pkcs12; diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs index 6e4697f4065..ce5c3be483d 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs @@ -1,8 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.DotNet.RemoteExecutor; +using Microsoft.DotNet.XUnitExtensions; +using Xunit; + namespace System.Security.Cryptography.X509Certificates.Tests { + [SkipOnPlatform(TestPlatforms.Browser, "Browser doesn't support X.509 certificates")] public class PfxIterationCountTests_X509Certificate2 : PfxIterationCountTests { internal override X509Certificate Import(byte[] blob) @@ -22,5 +27,29 @@ internal override X509Certificate Import(string fileName, string password) internal override X509Certificate Import(string fileName, SecureString password) => new X509Certificate2(fileName, password); + + + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void Import_IterationCountLimitExceeded_ThrowsInAllottedTime() + { + const int AllottedTime = 5000; + + if (!PfxTests.Pkcs12PBES2Supported) + { + throw new SkipTestException("Pkcs12NoPassword100MRounds uses PBES2, which is not supported on this version."); + } + + RemoteInvokeOptions options = new() + { + TimeOut = AllottedTime + }; + + RemoteExecutor.Invoke(static () => + { + byte[] blob = TestData.Pkcs12NoPassword100MRounds; + CryptographicException ce = Assert.Throws(() => new X509Certificate2(blob)); + Assert.Contains(FwlinkId, ce.Message); + }, options).Dispose(); + } } } diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/TestData.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/TestData.cs index 0d7633f9992..0e0295ecf7f 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/TestData.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/TestData.cs @@ -3437,6 +3437,26 @@ internal static DSAParameters GetDSA1024Params() "04020105000420AD0EB570ACFB8357A8E99B17672353CFBA69C76FFE5B6BC113" + "05577F12AE24040408D04E60444B79672302030927C1").HexToByteArray(); + internal static readonly byte[] Pkcs12NoPassword100MRounds = Convert.FromBase64String( + "MIIDygIBAzCCA4QGCSqGSIb3DQEHAaCCA3UEggNxMIIDbTCCA2kGCSqGSIb3DQEHBqCCA1owggNW" + + "AgEAMIIDTwYJKoZIhvcNAQcBMF4GCSqGSIb3DQEFDTBRMDAGCSqGSIb3DQEFDDAjBBCNparJkj/3" + + "Uk8N7n0KCMeQAgEBMAwGCCqGSIb3DQILBQAwHQYJYIZIAWUDBAEqBBAcqpBrSDFcXYAWVWKcsEi9" + + "gIIC4P/ANdPYWI1vBH1U5sZGMIwLjY96pYaBelyZd0ZfKA8QfGHVNP9+E9hplBKGvRfIMiqmFutj" + + "RO4v7Ls8HZEk0hwBt9+6zXPWDJLxBDfSMHUd08+ZAH1yzEqq8aBMyIRVHOQkJFuFuCQJ9Ke5HzVi" + + "39S1rgHpnKYFvy+xZAhgI9OO1YxuFt4P9nhlEV/JCoyEQ/2iY99kKc3z7ArrV7BBFhfYGKhWQCBu" + + "kAmNBKweRldNWgDuW21WJEl5sByOmyDwpiK55Zxy1K1aIY8DYJTtIzzcX4CILaj6tClMH1G9w4jW" + + "BkQI2CG4vCsMl/28BbIP9EyH2C+gBAxvc1N32y3NSvO0/GPVenmQFF9KBMc4FVy4Z21syMKzUkBi" + + "PtIbDkcQbGAfyPgFk4SXCgn8OpIIvOOGI50/r+Hj14qex9VIrlwAAWCH8Y+YjwqFAQJYHQpb47zp" + + "B1fTwJFOrsXrBgLUzJLZKLR43yW2E9u6b8RsTuFHjh985naCHLuWPYOXS1zduBpHKpwoPUyCwD2r" + + "DAokCvA7RCsSXroUkpJarN4CAqsEB8COnzV1Dl2xcAYMerJxrTCKX6WIQUYo0/qeCoqTT38lDAlE" + + "7Ydjyx12iVM6eWejAdjORvlVtCQQtCxz8fZpdFGbMP8rf35A8hu++e4u0CLHnhTx38zPIm6H6YfN" + + "qj5h1Kz0xLzqnRfa7EGfDEERSHOy/DqNY4nUNG2DTjGOHy1QJelToG7Vo2L7CCZV+leX0nwLNExf" + + "hKEp+uQCiYSJe9iDm9fS9VymED79OJbr2bxdq3MggEGksLZv/H0ZT8Wsue0vq9jQ6J6YIEM+DKYr" + + "Zt2l4WgTBEKbpqmRvOqYRh9O8Sp+3IRNPzMC2ehzlYXqoPbtG4vxpoRsAMCM/W2x61jbsBSaNSFA" + + "eaUwcnKswRg30UonHUAIOJkqtadI57WE/Rat5eHVyya9f7ZN8bTFZjx0BQs6Bo8PK9yfqoidSN8w" + + "PTAfMAcGBSsOAwIaBBTt8zpgzygINykjoAwr2GKEywYFwgQUA+L1vfCVASwiE++gTfRgIScMGycC" + + "BAX14QA="); + internal static readonly byte[] Pkcs12OpenSslOneCertDefaultEmptyPassword = ("308209CF0201033082098506092A864886F70D010701A0820976048209723082" + "096E308203E206092A864886F70D010706A08203D3308203CF020100308203C8" + diff --git a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_x509.c b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_x509.c index 8335c1c362c..94e22f28c87 100644 --- a/src/native/libs/System.Security.Cryptography.Native.Apple/pal_x509.c +++ b/src/native/libs/System.Security.Cryptography.Native.Apple/pal_x509.c @@ -70,7 +70,6 @@ PAL_X509ContentType AppleCryptoNative_X509GetContentType(uint8_t* pbData, int32_ // The sniffing order is: // * X509 DER // * PKCS7 PEM/DER - // * PKCS12 DER (or PEM if Apple has non-standard support for that) // * X509 PEM or PEM aggregate (or DER, but that already matched) // // If the X509 PEM check is done first SecItemImport will erroneously match @@ -78,6 +77,11 @@ PAL_X509ContentType AppleCryptoNative_X509GetContentType(uint8_t* pbData, int32_ // // Likewise, if the X509 DER check isn't done first, Apple will report it as // being a PKCS#7. + // + // This does not attempt to open a PFX / PKCS12 as Apple does not provide + // a suitable API to determine if it is PKCS12 without doing potentially + // unbound MAC / KDF work. Instead, let that return Unknown and let the managed + // decoding do the check. SecCertificateRef certref = SecCertificateCreateWithData(NULL, cfData); if (certref != NULL) @@ -104,41 +108,6 @@ PAL_X509ContentType AppleCryptoNative_X509GetContentType(uint8_t* pbData, int32_ } } - dataFormat = kSecFormatPKCS12; - actualFormat = dataFormat; - itemType = kSecItemTypeAggregate; - actualType = itemType; - - osStatus = SecItemImport(cfData, NULL, &actualFormat, &actualType, 0, NULL, NULL, NULL); - - if (osStatus == errSecPassphraseRequired) - { - dataFormat = kSecFormatPKCS12; - actualFormat = dataFormat; - itemType = kSecItemTypeAggregate; - actualType = itemType; - - SecItemImportExportKeyParameters importParams; - memset(&importParams, 0, sizeof(SecItemImportExportKeyParameters)); - - importParams.version = SEC_KEY_IMPORT_EXPORT_PARAMS_VERSION; - importParams.passphrase = CFSTR(""); - - osStatus = SecItemImport(cfData, NULL, &actualFormat, &actualType, 0, &importParams, NULL, NULL); - - CFRelease(importParams.passphrase); - importParams.passphrase = NULL; - } - - if (osStatus == noErr || osStatus == errSecPkcs12VerifyFailure) - { - if (actualType == itemType && actualFormat == dataFormat) - { - CFRelease(cfData); - return PAL_Pkcs12; - } - } - dataFormat = kSecFormatX509Cert; actualFormat = dataFormat; itemType = kSecItemTypeCertificate; From 0de70233bd5b314e2923540a03f50820c47f0f2d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 10 Apr 2024 10:38:45 -0700 Subject: [PATCH 052/151] [release/8.0-staging] Update CI builds to not use VS Previews (#100622) * Update CI to use Release VS2022. * fixing incorrect deletion --------- Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com> --- eng/pipelines/common/xplat-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 1f0c1016695..eb19570aeec 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -168,12 +168,12 @@ jobs: # Official Build Windows Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 # Public Windows Build Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), eq(variables['System.TeamProject'], 'public')) }}: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64.open + demands: ImageOverride -equals windows.vs2022.amd64.open ${{ if eq(parameters.helixQueuesTemplate, '') }}: From 4be29ad2fea7233f0e3397bfa8ca85ba3ef75ba1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:02:41 -0700 Subject: [PATCH 053/151] Update dependencies from https://github.com/dotnet/hotreload-utils build 20240410.1 (#100892) Microsoft.DotNet.HotReload.Utils.Generator.BuildTool From Version 8.0.0-alpha.0.24072.2 -> To Version 8.0.0-alpha.0.24210.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7bad61f7c23..5705ffaa791 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -354,9 +354,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization 67613417f5e1af250e6ddfba79f8f2885d8e90fb - + https://github.com/dotnet/hotreload-utils - bc857c64c5c5f1fc73048261e8f471c3310224d2 + 85d6e21ac1d4e0977dfd1321131be0c912d70d80 https://github.com/dotnet/runtime-assets diff --git a/eng/Versions.props b/eng/Versions.props index 609fc2931ba..efa8e0d8120 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -186,7 +186,7 @@ 8.0.0-prerelease.24112.2 8.0.0-prerelease.24112.2 8.0.0-prerelease.24112.2 - 8.0.0-alpha.0.24072.2 + 8.0.0-alpha.0.24210.1 2.4.2 1.0.0 2.4.5 From cca20d0d1bb6d9ad4f2d090f1348326df5cd158b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:04:55 -0700 Subject: [PATCH 054/151] [release/8.0-staging] Update dependencies from dotnet/runtime-assets (#100073) * Update dependencies from https://github.com/dotnet/runtime-assets build Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240320.9 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24115.2 -> To Version 8.0.0-beta.24170.9 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 56 ++++++++++++++++++++--------------------- eng/Versions.props | 28 ++++++++++----------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5705ffaa791..199869d6638 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -185,57 +185,57 @@ https://github.com/dotnet/arcade da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 https://github.com/dotnet/llvm-project @@ -358,9 +358,9 @@ https://github.com/dotnet/hotreload-utils 85d6e21ac1d4e0977dfd1321131be0c912d70d80 - + https://github.com/dotnet/runtime-assets - 0827f89642a6443c96a87990e3b910f6aa4cc4ed + 7b59ef83974ea3c51d746daf415e9ea9991f9519 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index efa8e0d8120..d5733206441 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,20 +143,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 - 8.0.0-beta.24115.2 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 + 8.0.0-beta.24170.9 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 From f37eb4819791a97b31fc6e29bae44589c0f88319 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:39:59 -0700 Subject: [PATCH 055/151] Update dependencies from https://github.com/dotnet/source-build-externals build 20240311.1 (#99601) Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24161.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ba7e1dd064f..9c16befab48 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 453a37ef7ae6c335cd49b3b9ab7713c87faeb265 - + https://github.com/dotnet/source-build-externals - 7a9b99e457a2b9792a3c17ccaf95d80038725108 + 00fb7841c80b44262646e57bcfbe90a1b7bc3151 From 0a26d20196959c00f58cc3c6146e51fc27ad97c8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:47:49 -0700 Subject: [PATCH 056/151] Update dependencies from https://github.com/dotnet/xharness build 20240408.4 (#100883) Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Common , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 8.0.0-prerelease.24112.2 -> To Version 8.0.0-prerelease.24208.4 Co-authored-by: dotnet-maestro[bot] Co-authored-by: Eric StJohn --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 6ff1559a00a..deb9fb7b21b 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "8.0.0-prerelease.24112.2", + "version": "8.0.0-prerelease.24208.4", "commands": [ "xharness" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9c16befab48..04d3d4429ae 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -322,17 +322,17 @@ https://github.com/dotnet/runtime edbd5c769a19798b6955050baccf99e6797d3208 - + https://github.com/dotnet/xharness - c055cc57f21796e79ace4bca2b070a8777f2446a + 9d21162000c444b2da3d6cdd805d43e1af51453a - + https://github.com/dotnet/xharness - c055cc57f21796e79ace4bca2b070a8777f2446a + 9d21162000c444b2da3d6cdd805d43e1af51453a - + https://github.com/dotnet/xharness - c055cc57f21796e79ace4bca2b070a8777f2446a + 9d21162000c444b2da3d6cdd805d43e1af51453a https://github.com/dotnet/arcade diff --git a/eng/Versions.props b/eng/Versions.props index f6572992a76..65b9a961153 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -183,9 +183,9 @@ 1.1.0 17.4.0-preview-20220707-01 - 8.0.0-prerelease.24112.2 - 8.0.0-prerelease.24112.2 - 8.0.0-prerelease.24112.2 + 8.0.0-prerelease.24208.4 + 8.0.0-prerelease.24208.4 + 8.0.0-prerelease.24208.4 8.0.0-alpha.0.24210.1 2.4.2 1.0.0 From 24e52a017380186e436457c80651e2cdec1b0095 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 08:46:02 -0700 Subject: [PATCH 057/151] [release/8.0-staging] Update dependencies from dotnet/source-build-reference-packages (#99902) * Update dependencies from https://github.com/dotnet/source-build-reference-packages build Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24061.1 -> To Version 8.0.0-alpha.1.24163.3 * Update dependencies from https://github.com/dotnet/source-build-reference-packages build Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24061.1 -> To Version 8.0.0-alpha.1.24163.3 * Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240313.3 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24061.1 -> To Version 8.0.0-alpha.1.24163.3 * Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240313.3 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24061.1 -> To Version 8.0.0-alpha.1.24163.3 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 04d3d4429ae..2f0eb24fa19 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -99,9 +99,9 @@ 08a90ca2c88b17f1b5d081318354a41db0882cff - + https://github.com/dotnet/source-build-reference-packages - 453a37ef7ae6c335cd49b3b9ab7713c87faeb265 + 79827eed138fd2575a8b24820b4f385ee4ffb6e6 From b1b775e58c07060150e75e73779661f8949524d2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 09:04:59 -0700 Subject: [PATCH 058/151] [release/8.0-staging] Update dependencies from dotnet/source-build-externals (#100015) * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24168.2 * Update dependencies from https://github.com/dotnet/source-build-externals build Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24158.3 -> To Version 8.0.0-alpha.1.24175.3 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240325.3 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24161.1 -> To Version 8.0.0-alpha.1.24175.3 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Eric StJohn --- NuGet.config | 2 ++ eng/Version.Details.xml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/NuGet.config b/NuGet.config index 50349586bd3..86aa3c4ed58 100644 --- a/NuGet.config +++ b/NuGet.config @@ -10,6 +10,8 @@ + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2f0eb24fa19..7fabe9f4f99 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 79827eed138fd2575a8b24820b4f385ee4ffb6e6 - + https://github.com/dotnet/source-build-externals - 00fb7841c80b44262646e57bcfbe90a1b7bc3151 + 300e99190e6ae1983681694dbdd5f75f0c692081 From be4864fe7e2c46d60ca2bfaeb4495ccc6568c610 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 09:07:03 -0700 Subject: [PATCH 059/151] Update dependencies from https://github.com/dotnet/runtime-assets build 20240411.2 (#100958) Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24170.9 -> To Version 8.0.0-beta.24211.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 56 ++++++++++++++++++++--------------------- eng/Versions.props | 28 ++++++++++----------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7fabe9f4f99..be15ab04c85 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -185,57 +185,57 @@ https://github.com/dotnet/arcade da98edc4c3ea539f109ea320672136ceb32591a7 - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d https://github.com/dotnet/llvm-project @@ -358,9 +358,9 @@ https://github.com/dotnet/hotreload-utils 85d6e21ac1d4e0977dfd1321131be0c912d70d80 - + https://github.com/dotnet/runtime-assets - 7b59ef83974ea3c51d746daf415e9ea9991f9519 + dc8ac7417ae48bfbf2516a346723107af0ad603d https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 65b9a961153..b1ee52a12eb 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,20 +143,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 - 8.0.0-beta.24170.9 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 + 8.0.0-beta.24211.2 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 From 78cde8759043a7f2b550c7278cfb40d9909071dc Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 09:53:51 -0700 Subject: [PATCH 060/151] Update dependencies from https://github.com/dotnet/arcade build 20240404.3 (#100885) Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.24113.2 -> To Version 8.0.0-beta.24204.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 72 ++--- eng/Versions.props | 30 +- eng/common/SetupNugetSources.ps1 | 26 +- eng/common/templates-official/job/job.yml | 264 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 155 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 68 +++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 22 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../templates-official/steps/build-reason.yml | 12 + .../steps/component-governance.yml | 13 + .../steps/execute-codeql.yml | 32 ++ .../templates-official/steps/execute-sdl.yml | 88 ++++++ .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + eng/common/templates/job/job.yml | 4 + .../templates/steps/component-governance.yml | 2 +- eng/common/templates/steps/generate-sbom.yml | 2 +- global.json | 6 +- 31 files changed, 1832 insertions(+), 69 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/build-reason.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/execute-sdl.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index be15ab04c85..f91d9947758 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -111,9 +111,9 @@ - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba @@ -121,69 +121,69 @@ 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba https://github.com/dotnet/runtime-assets @@ -334,9 +334,9 @@ https://github.com/dotnet/xharness 9d21162000c444b2da3d6cdd805d43e1af51453a - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 188340e12c0a372b1681ad6a5e72c608021efdba https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index b1ee52a12eb..afc5da2fc7d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,21 +87,21 @@ 8.0.100 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 2.5.1-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 - 8.0.0-beta.24113.2 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 2.5.1-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 + 8.0.0-beta.24204.3 6.0.0-preview.1.102 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925f..efa2fd72bfa 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 00000000000..1f035fee73f --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,264 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 00000000000..52b4d05d3f8 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 00000000000..589ac80a18b --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,155 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 00000000000..f193dfbe236 --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2 + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 00000000000..f0513aee5b0 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 00000000000..b68d3c2f319 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 00000000000..857a0f8ba43 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 00000000000..08e5db9bb11 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 00000000000..c24193acfc9 --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 00000000000..da1f40958b4 --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..0c87f149a4a --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 00000000000..da669030daf --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 00000000000..f67a210d62f --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 00000000000..eba58109b52 --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 00000000000..cbba0596709 --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 00000000000..9b4a5ffa30a --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 00000000000..07426fde05d --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 00000000000..1bf43bf807a --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 00000000000..04012fed182 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 00000000000..83d97a26a01 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 00000000000..3eb7e2d5f84 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 00000000000..829f17c34d1 --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 00000000000..1f308b24efc --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022 + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 00000000000..dbdd66d4a4b --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f9..8ec5c4f2d9f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c9..cbba0596709 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa..2b21eae4273 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/global.json b/global.json index c987cd4bc00..e0a39423109 100644 --- a/global.json +++ b/global.json @@ -8,9 +8,9 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24113.2", - "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24113.2", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24204.3", + "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24204.3", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "8.0.0-rc.1.23406.6" From 9b82dcc5e647c503802a41bb00036386560466c1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 10:05:45 -0700 Subject: [PATCH 061/151] Fix native leak in CryptoNative_GetX509nameInfo The implementation was previously only freeing the STACK_OF, but not the contents of the stack. This change fixes not freeing the individual GENERAL_NAME items. (#100793) Co-authored-by: Kevin Jones --- .../libs/System.Security.Cryptography.Native/openssl.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/native/libs/System.Security.Cryptography.Native/openssl.c b/src/native/libs/System.Security.Cryptography.Native/openssl.c index 73822045895..fa2302b780e 100644 --- a/src/native/libs/System.Security.Cryptography.Native/openssl.c +++ b/src/native/libs/System.Security.Cryptography.Native/openssl.c @@ -627,8 +627,8 @@ BIO* CryptoNative_GetX509NameInfo(X509* x509, int32_t nameType, int32_t forIssue break; } - STACK_OF(GENERAL_NAME)* altNames = (STACK_OF(GENERAL_NAME)*)( - X509_get_ext_d2i(x509, forIssuer ? NID_issuer_alt_name : NID_subject_alt_name, NULL, NULL)); + GENERAL_NAMES* altNames = (GENERAL_NAMES*) + X509_get_ext_d2i(x509, forIssuer ? NID_issuer_alt_name : NID_subject_alt_name, NULL, NULL); if (altNames) { @@ -686,13 +686,13 @@ BIO* CryptoNative_GetX509NameInfo(X509* x509, int32_t nameType, int32_t forIssue { BIO* b = BIO_new(BIO_s_mem()); ASN1_STRING_print_ex(b, str, ASN1_STRFLGS_UTF8_CONVERT); - sk_GENERAL_NAME_free(altNames); + GENERAL_NAMES_free(altNames); return b; } } } - sk_GENERAL_NAME_free(altNames); + GENERAL_NAMES_free(altNames); } } From 7fa8815b1a2bc5183a0ea4549b4aad8d2e82acd6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 12 Apr 2024 18:01:15 -0500 Subject: [PATCH 062/151] [release/8.0-staging] Update dependencies from dotnet/emsdk (#99696) * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24163.1 * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24165.3 * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24170.6 * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24172.5 * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24175.3 * Update dependencies from https://github.com/dotnet/emsdk build Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24176.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240327.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24177.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240327.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24177.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240328.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24178.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240329.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.4-servicing.24179.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240403.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24203.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240405.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24205.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240409.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24209.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240410.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24210.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240411.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24211.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240411.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24211.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240411.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24156.2 -> To Version 8.0.5-servicing.24211.3 * Update dependencies from https://github.com/dotnet/emsdk build 20240411.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.4-servicing.24163.1 -> To Version 8.0.5-servicing.24211.3 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Eric StJohn --- NuGet.config | 4 +--- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/NuGet.config b/NuGet.config index 86aa3c4ed58..a0aef486d23 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,9 +9,7 @@ - - - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f91d9947758..91df0c87d9c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 08a90ca2c88b17f1b5d081318354a41db0882cff + 71359b18c2d83c01a68bf155244a65962a7e8c8e - + https://github.com/dotnet/emsdk - 08a90ca2c88b17f1b5d081318354a41db0882cff + 71359b18c2d83c01a68bf155244a65962a7e8c8e diff --git a/eng/Versions.props b/eng/Versions.props index afc5da2fc7d..d0418b1a22e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -240,7 +240,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.4 + 8.0.5 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From f9feb6865ffbbbdb213bfe75229b431932b28a0f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 13 Apr 2024 17:48:22 -0500 Subject: [PATCH 063/151] Try to find seqpoints directly from jitinfo if it's not find on get_default_jit_mm (#100978) Co-authored-by: Thays Grazia --- src/mono/mono/component/debugger-engine.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/mono/mono/component/debugger-engine.c b/src/mono/mono/component/debugger-engine.c index 5ec3e68b62a..9a3648b311b 100644 --- a/src/mono/mono/component/debugger-engine.c +++ b/src/mono/mono/component/debugger-engine.c @@ -449,6 +449,16 @@ mono_de_set_breakpoint (MonoMethod *method, long il_offset, EventRequest *req, M set_bp_in_method (domain, m, seq_points, bp, error); } + // trying to get the seqpoints directly from the jit info of the method + // the seqpoints in get_default_jit_mm may not be found for AOTed methods in arm64 + if (methods->len == 0) + { + MonoJitInfo *ji; + (void)mono_jit_search_all_backends_for_jit_info (method, &ji); + if (ji && ji->seq_points) + set_bp_in_method (mono_get_root_domain (), method, ji->seq_points, bp, error); + } + g_ptr_array_add (breakpoints, bp); mono_debugger_log_add_bp (bp, bp->method, bp->il_offset); mono_loader_unlock (); From af764fd245c4a90b59ac8b29ad9ae95f705beac4 Mon Sep 17 00:00:00 2001 From: Pavel Savara Date: Mon, 15 Apr 2024 16:29:49 +0200 Subject: [PATCH 064/151] [release/8.0][browser] fix emscripten out/err overrides (#100818) * backport of https://github.com/dotnet/runtime/pull/100630/ * fix * fix --- .../scenarios/BuildWasmAppsJobsList.txt | 2 +- src/mono/sample/wasm/browser-advanced/main.js | 1 + ...eProgressTests.cs => ModuleConfigTests.cs} | 26 +++++++++++++++-- src/mono/wasm/runtime/loader/assets.ts | 3 +- src/mono/wasm/runtime/loader/icu.ts | 3 +- src/mono/wasm/runtime/loader/run.ts | 2 +- src/mono/wasm/runtime/startup.ts | 29 +++++++++++-------- src/mono/wasm/runtime/types/internal.ts | 6 ++-- .../WasmBasicTestApp/App/Common/Program.cs | 1 + .../WasmBasicTestApp/App/wwwroot/main.js | 15 ++++++++++ 10 files changed, 67 insertions(+), 21 deletions(-) rename src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/{DownloadResourceProgressTests.cs => ModuleConfigTests.cs} (68%) diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index 51bb6bcbb3f..e16c74fe4f0 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -41,5 +41,5 @@ Wasm.Build.Tests.WasmRunOutOfAppBundleTests Wasm.Build.Tests.WasmSIMDTests Wasm.Build.Tests.WasmTemplateTests Wasm.Build.Tests.WorkloadTests -Wasm.Build.Tests.TestAppScenarios.DownloadResourceProgressTests +Wasm.Build.Tests.TestAppScenarios.ModuleConfigTests Wasm.Build.Tests.MT.Blazor.SimpleMultiThreadedTests diff --git a/src/mono/sample/wasm/browser-advanced/main.js b/src/mono/sample/wasm/browser-advanced/main.js index b5c414322fe..85b1a64e8e4 100644 --- a/src/mono/sample/wasm/browser-advanced/main.js +++ b/src/mono/sample/wasm/browser-advanced/main.js @@ -59,6 +59,7 @@ try { console.log('user code Module.onDotnetReady'); }, postRun: () => { console.log('user code Module.postRun'); }, + out: (text) => { console.log("ADVANCED:" + text) }, }) .withResourceLoader((type, name, defaultUri, integrity, behavior) => { // loadBootResource could return string with unqualified name of resource. It assumes that we resolve it with document.baseURI diff --git a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DownloadResourceProgressTests.cs b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/ModuleConfigTests.cs similarity index 68% rename from src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DownloadResourceProgressTests.cs rename to src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/ModuleConfigTests.cs index 70f9b4f1507..1b8c43ecea7 100644 --- a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DownloadResourceProgressTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/ModuleConfigTests.cs @@ -13,9 +13,9 @@ namespace Wasm.Build.Tests.TestAppScenarios; -public class DownloadResourceProgressTests : AppTestBase +public class ModuleConfigTests : AppTestBase { - public DownloadResourceProgressTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + public ModuleConfigTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) : base(output, buildContext) { } @@ -25,7 +25,7 @@ public DownloadResourceProgressTests(ITestOutputHelper output, SharedBuildPerTes [InlineData(true)] public async Task DownloadProgressFinishes(bool failAssemblyDownload) { - CopyTestAsset("WasmBasicTestApp", $"DownloadResourceProgressTests_{failAssemblyDownload}"); + CopyTestAsset("WasmBasicTestApp", $"ModuleConfigTests_DownloadProgressFinishes_{failAssemblyDownload}"); PublishProject("Debug"); var result = await RunSdkStyleApp(new( @@ -54,4 +54,24 @@ public async Task DownloadProgressFinishes(bool failAssemblyDownload) : "The download progress test did emit unexpected message about failing download" ); } + + [Fact] + public async Task OutErrOverrideWorks() + { + CopyTestAsset("WasmBasicTestApp", $"ModuleConfigTests_OutErrOverrideWorks"); + PublishProject("Debug"); + + var result = await RunSdkStyleApp(new( + Configuration: "Debug", + TestScenario: "OutErrOverrideWorks" + )); + Assert.True( + result.ConsoleOutput.Any(m => m.Contains("Emscripten out override works!")), + "Emscripten out override doesn't work" + ); + Assert.True( + result.ConsoleOutput.Any(m => m.Contains("Emscripten err override works!")), + "Emscripten err override doesn't work" + ); + } } diff --git a/src/mono/wasm/runtime/loader/assets.ts b/src/mono/wasm/runtime/loader/assets.ts index 8730988193f..5e9eb8a8d3b 100644 --- a/src/mono/wasm/runtime/loader/assets.ts +++ b/src/mono/wasm/runtime/loader/assets.ts @@ -12,6 +12,7 @@ import { mono_exit } from "./exit"; import { addCachedReponse, findCachedResponse } from "./assetsCache"; import { getIcuResourceName } from "./icu"; import { makeURLAbsoluteWithApplicationBase } from "./polyfills"; +import { mono_log_info } from "./logging"; let throttlingPromise: PromiseAndController | undefined; @@ -545,7 +546,7 @@ async function start_asset_download_sources(asset: AssetEntryInternal): Promise< err.status = response.status; throw err; } else { - loaderHelpers.out(`optional download '${response.url}' for ${asset.name} failed ${response.status} ${response.statusText}`); + mono_log_info(`optional download '${response.url}' for ${asset.name} failed ${response.status} ${response.statusText}`); return undefined; } } diff --git a/src/mono/wasm/runtime/loader/icu.ts b/src/mono/wasm/runtime/loader/icu.ts index 927672f5db9..bb7213b22a8 100644 --- a/src/mono/wasm/runtime/loader/icu.ts +++ b/src/mono/wasm/runtime/loader/icu.ts @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +import { mono_log_error } from "./logging"; import { GlobalizationMode, MonoConfig } from "../types"; import { ENVIRONMENT_IS_WEB, loaderHelpers } from "./globals"; import { mono_log_info, mono_log_debug } from "./logging"; @@ -18,7 +19,7 @@ export function init_globalization() { loaderHelpers.preferredIcuAsset = null; } else { const msg = "invariant globalization mode is inactive and no ICU data archives are available"; - loaderHelpers.err(`ERROR: ${msg}`); + mono_log_error(`ERROR: ${msg}`); throw new Error(msg); } } diff --git a/src/mono/wasm/runtime/loader/run.ts b/src/mono/wasm/runtime/loader/run.ts index b3437a5a81d..112d66edab9 100644 --- a/src/mono/wasm/runtime/loader/run.ts +++ b/src/mono/wasm/runtime/loader/run.ts @@ -458,7 +458,7 @@ async function initializeModules(es6Modules: [RuntimeModuleExportsInternal, Nati const { default: emscriptenFactory } = es6Modules[1]; setRuntimeGlobals(globalObjectsRoot); initializeExports(globalObjectsRoot); - await configureRuntimeStartup(); + await configureRuntimeStartup(globalObjectsRoot.module); loaderHelpers.runtimeModuleLoaded.promise_control.resolve(); emscriptenFactory((originalModule: EmscriptenModuleInternal) => { diff --git a/src/mono/wasm/runtime/startup.ts b/src/mono/wasm/runtime/startup.ts index c385844dfd3..324bdc78b04 100644 --- a/src/mono/wasm/runtime/startup.ts +++ b/src/mono/wasm/runtime/startup.ts @@ -39,7 +39,23 @@ import { assertNoProxies } from "./gc-handles"; // default size if MonoConfig.pthreadPoolSize is undefined const MONO_PTHREAD_POOL_SIZE = 4; -export async function configureRuntimeStartup(): Promise { +export async function configureRuntimeStartup (module: DotnetModuleInternal): Promise { + if (!module.out) { + // eslint-disable-next-line no-console + module.out = console.log.bind(console); + } + if (!module.err) { + // eslint-disable-next-line no-console + module.err = console.error.bind(console); + } + if (!module.print) { + module.print = module.out; + } + if (!module.printErr) { + module.printErr = module.err; + } + loaderHelpers.out = module.print; + loaderHelpers.err = module.printErr; await init_polyfills_async(); await checkMemorySnapshotSize(); } @@ -54,17 +70,6 @@ export function configureEmscriptenStartup(module: DotnetModuleInternal): void { module.locateFile = module.__locateFile = (path) => loaderHelpers.scriptDirectory + path; } - if (!module.out) { - // eslint-disable-next-line no-console - module.out = console.log.bind(console); - } - - if (!module.err) { - // eslint-disable-next-line no-console - module.err = console.error.bind(console); - } - loaderHelpers.out = module.out; - loaderHelpers.err = module.err; module.mainScriptUrlOrBlob = loaderHelpers.scriptUrl;// this is needed by worker threads // these all could be overridden on DotnetModuleConfig, we are chaing them to async below, as opposed to emscripten diff --git a/src/mono/wasm/runtime/types/internal.ts b/src/mono/wasm/runtime/types/internal.ts index a91b21973c8..c43443a48da 100644 --- a/src/mono/wasm/runtime/types/internal.ts +++ b/src/mono/wasm/runtime/types/internal.ts @@ -464,6 +464,8 @@ export declare interface EmscriptenModuleInternal { runtimeKeepalivePush(): void; runtimeKeepalivePop(): void; maybeExit(): void; + print(message: string): void; + printErr(message: string): void; } /// A PromiseController encapsulates a Promise together with easy access to its resolve and reject functions. @@ -492,7 +494,7 @@ export type setGlobalObjectsType = (globalObjects: GlobalObjects) => void; export type initializeExportsType = (globalObjects: GlobalObjects) => RuntimeAPI; export type initializeReplacementsType = (replacements: EmscriptenReplacements) => void; export type configureEmscriptenStartupType = (module: DotnetModuleInternal) => void; -export type configureRuntimeStartupType = () => Promise; +export type configureRuntimeStartupType = (module: DotnetModuleInternal) => Promise; export type configureWorkerStartupType = (module: DotnetModuleInternal) => Promise @@ -508,4 +510,4 @@ export type RuntimeModuleExportsInternal = { export type NativeModuleExportsInternal = { default: (unificator: Function) => EmscriptenModuleInternal -} \ No newline at end of file +} diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/Common/Program.cs b/src/mono/wasm/testassets/WasmBasicTestApp/App/Common/Program.cs index 4f38c02031c..eda19b81f50 100644 --- a/src/mono/wasm/testassets/WasmBasicTestApp/App/Common/Program.cs +++ b/src/mono/wasm/testassets/WasmBasicTestApp/App/Common/Program.cs @@ -2,3 +2,4 @@ // The .NET Foundation licenses this file to you under the MIT license. System.Console.WriteLine("WasmBasicTestApp"); +System.Console.Error.WriteLine("WasmBasicTestApp stderr"); diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js index 076f37a62a6..8e3485bc2e6 100644 --- a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js +++ b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js @@ -63,6 +63,18 @@ switch (testCase) { } }); break; + case "OutErrOverrideWorks": + dotnet.withModuleConfig({ + out: (message) => { + console.log("Emscripten out override works!"); + console.log(message) + }, + err: (message) => { + console.error("Emscripten err override works!"); + console.error(message) + }, + }); + break; } const { getAssemblyExports, getConfig, INTERNAL } = await dotnet.create(); @@ -94,6 +106,9 @@ try { case "DownloadResourceProgressTest": exit(0); break; + case "OutErrOverrideWorks": + dotnet.run(); + break; default: console.error(`Unknown test case: ${testCase}`); exit(3); From cd27afea7a3621f9b14a46553b39b4550c1ace44 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:19:54 -0700 Subject: [PATCH 065/151] [mono][eventpipe] Fix firing dynamic method wrappers crash (#99712) Co-authored-by: mdh1418 --- src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c b/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c index ccb91d0ca41..4ab963643d4 100644 --- a/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c +++ b/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c @@ -779,6 +779,8 @@ include_method (MonoMethod *method) return false; } else if (!m_method_is_wrapper (method)) { return true; + } else if (method->wrapper_type == MONO_WRAPPER_DYNAMIC_METHOD){ + return true; } else { WrapperInfo *wrapper = mono_marshal_get_wrapper_info (method); return (wrapper && wrapper->subtype == WRAPPER_SUBTYPE_PINVOKE) ? true : false; From f0f42cc1f6af798c90c1da47d914e08969e3ac24 Mon Sep 17 00:00:00 2001 From: Rich Lander Date: Mon, 15 Apr 2024 11:22:29 -0700 Subject: [PATCH 066/151] Update Alpine versions for `release/8.0-staging` (#99849) * Update Alpine versions * Backport [#99020] Disable QUIC tests on alpine arm32 --------- Co-authored-by: Natalia Kondratyeva --- .../coreclr/templates/helix-queues-setup.yml | 12 ++++++------ eng/pipelines/libraries/helix-queues-setup.yml | 8 ++++---- .../FunctionalTests/MsQuicCipherSuitesPolicyTests.cs | 3 ++- .../FunctionalTests/MsQuicPlatformDetectionTests.cs | 1 + .../tests/FunctionalTests/MsQuicTests.cs | 1 + .../tests/FunctionalTests/QuicConnectionTests.cs | 1 + .../tests/FunctionalTests/QuicListenerTests.cs | 1 + .../QuicStreamConnectedStreamConformanceTests.cs | 1 + .../tests/FunctionalTests/QuicStreamTests.cs | 1 + 9 files changed, 18 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 8e5ddd36759..9a57f32f5e6 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -79,23 +79,23 @@ jobs: # Linux musl x64 - ${{ if eq(parameters.platform, 'linux_musl_x64') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.315.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-amd64 + - (Alpine.317.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-amd64 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.315.Amd64)ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-amd64 + - (Alpine.319.Amd64)Ubuntu.2204.Amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-helix-amd64 # Linux musl arm32 - ${{ if eq(parameters.platform, 'linux_musl_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.315.Arm32.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-arm32v7 + - (Alpine.317.Arm32.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm32v7 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.315.Arm32)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-arm32v7 + - (Alpine.319.Arm32)Ubuntu.2204.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-helix-arm32v7 # Linux musl arm64 - ${{ if eq(parameters.platform, 'linux_musl_arm64') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.315.Arm64.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-arm64v8 + - (Alpine.317.Arm64.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm64v8 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.315.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-arm64v8 + - (Alpine.319.Arm64)Ubuntu.2204.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-helix-arm64v8 # Linux x64 - ${{ if eq(parameters.platform, 'linux_x64') }}: diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 5b37825df08..be98c70f161 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -47,14 +47,14 @@ jobs: # Linux musl x64 - ${{ if eq(parameters.platform, 'linux_musl_x64') }}: - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Alpine.315.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-amd64 + - (Alpine.317.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-amd64 - ${{ if or(eq(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Alpine.317.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-amd64 + - (Alpine.319.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-helix-amd64 # Linux musl arm64 - ${{ if and(eq(parameters.platform, 'linux_musl_arm64'), or(eq(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true))) }}: - - (Alpine.317.Arm64.Open)ubuntu.1804.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm64v8 - - (Alpine.315.Arm64.Open)ubuntu.1804.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.15-helix-arm64v8 + - (Alpine.317.Arm64.Open)ubuntu.2204.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm64v8 + - (Alpine.319.Arm64.Open)ubuntu.2204.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.19-helix-arm64v8 # Linux x64 - ${{ if eq(parameters.platform, 'linux_x64') }}: diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs index df827e64fea..8d71bfc4cb0 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs @@ -11,6 +11,7 @@ namespace System.Net.Quic.Tests [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] [SkipOnPlatform(TestPlatforms.Windows, "CipherSuitesPolicy is not supported on Windows")] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public class MsQuicCipherSuitesPolicyTests : QuicTestBase { public MsQuicCipherSuitesPolicyTests(ITestOutputHelper output) : base(output) { } @@ -77,4 +78,4 @@ await Assert.ThrowsAsync(() => TestConnection( )); } } -} \ No newline at end of file +} diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs index 16f267fc719..8e61c741a92 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs @@ -59,6 +59,7 @@ public async Task SupportedLinuxPlatformsWithMsQuic_IsSupportedIsTrue() [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsRaspbian10), nameof(PlatformDetection.IsArmv6Process), nameof(PlatformDetection.IsInContainer))] [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsPpc64leProcess))] [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsUbuntu2004), nameof(PlatformDetection.IsS390xProcess))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsInHelix))] [PlatformSpecific(TestPlatforms.Linux)] public void SupportedLinuxPlatforms_IsSupportedIsTrue() diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs index 503c2d40683..51c4279308e 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs @@ -48,6 +48,7 @@ public void Dispose() [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public class MsQuicTests : QuicTestBase, IClassFixture { private static byte[] s_data = "Hello world!"u8.ToArray(); diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs index dba45d81394..01dbaa5fb9f 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs @@ -16,6 +16,7 @@ namespace System.Net.Quic.Tests [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicConnectionTests : QuicTestBase { const int ExpectedErrorCode = 1234; diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs index 68509bf6b55..3abc21344c9 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs @@ -15,6 +15,7 @@ namespace System.Net.Quic.Tests { [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicListenerTests : QuicTestBase { public QuicListenerTests(ITestOutputHelper output) : base(output) { } diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs index 762fc4230ab..63f577cbc69 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs @@ -16,6 +16,7 @@ namespace System.Net.Quic.Tests { [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicStreamConformanceTests : ConnectedStreamConformanceTests { protected override bool UsableAfterCanceledReads => false; diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs index e220decb1bd..dabd0a6e401 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs @@ -14,6 +14,7 @@ namespace System.Net.Quic.Tests { [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicStreamTests : QuicTestBase { private static byte[] s_data = "Hello world!"u8.ToArray(); From 6f292679f9973fa80b242b6d39da1330c2ef7de9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 11:25:02 -0700 Subject: [PATCH 067/151] [release/8.0-staging] Guard against -1 Returned from sysconf for the Cache Sizes Causing Large Gen0 Sizes and Budgets for Certain Linux Distributions. (#100575) * Logging. * Fixed comparison check * Fix logical operations * Completely guard against the cacheSize as UINTMAX_MAX * Fix for right macro * Ensure we are guarded against all cases where cacheSize == SIZE_MAX * Added an extra guard and removed redundant case * Comment clean * Added some additional asserts * Removed unnecessary checks for cacheSize == SIZE_MAX * Cleaned up logic * Fix type casting comparison * Removed redundant comment * Removed one more unneccesary guard --------- Co-authored-by: mrsharm --- src/coreclr/gc/unix/gcenv.unix.cpp | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp index b45cd40d807..dd591a54426 100644 --- a/src/coreclr/gc/unix/gcenv.unix.cpp +++ b/src/coreclr/gc/unix/gcenv.unix.cpp @@ -792,28 +792,30 @@ bool ReadMemoryValueFromFile(const char* filename, uint64_t* val) return result; } -#define UPDATE_CACHE_SIZE_AND_LEVEL(NEW_CACHE_SIZE, NEW_CACHE_LEVEL) if (NEW_CACHE_SIZE > cacheSize) { cacheSize = NEW_CACHE_SIZE; cacheLevel = NEW_CACHE_LEVEL; } +#define UPDATE_CACHE_SIZE_AND_LEVEL(NEW_CACHE_SIZE, NEW_CACHE_LEVEL) if (NEW_CACHE_SIZE > ((long)cacheSize)) { cacheSize = NEW_CACHE_SIZE; cacheLevel = NEW_CACHE_LEVEL; } static size_t GetLogicalProcessorCacheSizeFromOS() { size_t cacheLevel = 0; size_t cacheSize = 0; - size_t size; + long size; + // sysconf can return -1 if the cache size is unavailable in some distributions and 0 in others. + // UPDATE_CACHE_SIZE_AND_LEVEL should handle both the cases by not updating cacheSize if either of cases are met. #ifdef _SC_LEVEL1_DCACHE_SIZE - size = ( size_t) sysconf(_SC_LEVEL1_DCACHE_SIZE); + size = sysconf(_SC_LEVEL1_DCACHE_SIZE); UPDATE_CACHE_SIZE_AND_LEVEL(size, 1) #endif #ifdef _SC_LEVEL2_CACHE_SIZE - size = ( size_t) sysconf(_SC_LEVEL2_CACHE_SIZE); + size = sysconf(_SC_LEVEL2_CACHE_SIZE); UPDATE_CACHE_SIZE_AND_LEVEL(size, 2) #endif #ifdef _SC_LEVEL3_CACHE_SIZE - size = ( size_t) sysconf(_SC_LEVEL3_CACHE_SIZE); + size = sysconf(_SC_LEVEL3_CACHE_SIZE); UPDATE_CACHE_SIZE_AND_LEVEL(size, 3) #endif #ifdef _SC_LEVEL4_CACHE_SIZE - size = ( size_t) sysconf(_SC_LEVEL4_CACHE_SIZE); + size = sysconf(_SC_LEVEL4_CACHE_SIZE); UPDATE_CACHE_SIZE_AND_LEVEL(size, 4) #endif @@ -836,17 +838,22 @@ static size_t GetLogicalProcessorCacheSizeFromOS() { path_to_size_file[index] = (char)(48 + i); - if (ReadMemoryValueFromFile(path_to_size_file, &size)) + uint64_t cache_size_from_sys_file = 0; + + if (ReadMemoryValueFromFile(path_to_size_file, &cache_size_from_sys_file)) { + // uint64_t to long conversion as ReadMemoryValueFromFile takes a uint64_t* as an argument for the val argument. + size = (long)cache_size_from_sys_file; path_to_level_file[index] = (char)(48 + i); if (ReadMemoryValueFromFile(path_to_level_file, &level)) { UPDATE_CACHE_SIZE_AND_LEVEL(size, level) } + else { - cacheSize = std::max(cacheSize, size); + cacheSize = std::max((long)cacheSize, size); } } } From a494d22d2b50bf7811549ac26ee459aed0af6c4f Mon Sep 17 00:00:00 2001 From: Jan Vorlicek Date: Mon, 15 Apr 2024 20:26:53 +0200 Subject: [PATCH 068/151] [Release/8.0] Remove preventing EH at shutdown (#100836) Partial backport of #100293 to release/8.0 --- src/coreclr/vm/ceemain.cpp | 3 --- src/coreclr/vm/eepolicy.cpp | 7 ------- src/coreclr/vm/excep.cpp | 12 ++---------- src/coreclr/vm/i386/excepx86.cpp | 3 --- src/coreclr/vm/vars.cpp | 1 - src/coreclr/vm/vars.hpp | 1 - 6 files changed, 2 insertions(+), 25 deletions(-) diff --git a/src/coreclr/vm/ceemain.cpp b/src/coreclr/vm/ceemain.cpp index 218cda11d7f..680a9a2fb24 100644 --- a/src/coreclr/vm/ceemain.cpp +++ b/src/coreclr/vm/ceemain.cpp @@ -1331,9 +1331,6 @@ void STDMETHODCALLTYPE EEShutDownHelper(BOOL fIsDllUnloading) // Shutdown finalizer before we suspend all background threads. Otherwise we // never get to finalize anything. - // No longer process exceptions - g_fNoExceptions = true; - // @TODO: This does things which shouldn't occur in part 2. Namely, // calling managed dll main callbacks (AppDomain::SignalProcessDetach), and // RemoveAppDomainFromIPC. diff --git a/src/coreclr/vm/eepolicy.cpp b/src/coreclr/vm/eepolicy.cpp index af27338544e..6069f0ec124 100644 --- a/src/coreclr/vm/eepolicy.cpp +++ b/src/coreclr/vm/eepolicy.cpp @@ -59,13 +59,6 @@ void SafeExitProcess(UINT exitCode, ShutdownCompleteAction sca = SCA_ExitProcess } } - // Turn off exception processing, because if some other random DLL has a - // fault in DLL_PROCESS_DETACH, we could get called for exception handling. - // Since we've turned off part of the runtime, we can't, for instance, - // properly execute the GC that handling an exception might trigger. - g_fNoExceptions = true; - LOG((LF_EH, LL_INFO10, "SafeExitProcess: turning off exceptions\n")); - if (sca == SCA_TerminateProcessWhenShutdownComplete) { // disabled because if we fault in this code path we will trigger our Watson code diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index ce8b325a5f4..b88d6a959bb 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -4623,12 +4623,6 @@ LONG InternalUnhandledExceptionFilter_Worker( } #endif - // This shouldn't be possible, but MSVC re-installs us... for now, just bail if this happens. - if (g_fNoExceptions) - { - return EXCEPTION_CONTINUE_SEARCH; - } - // Are we looking at a stack overflow here? if ((pThread != NULL) && !pThread->DetermineIfGuardPagePresent()) { @@ -5533,8 +5527,6 @@ static LONG ThreadBaseExceptionFilter_Worker(PEXCEPTION_POINTERS pExceptionInfo, ThreadBaseExceptionFilterParam *pParam = (ThreadBaseExceptionFilterParam *) pvParam; UnhandledExceptionLocation location = pParam->location; - _ASSERTE(!g_fNoExceptions); - Thread* pThread = GetThread(); #ifdef _DEBUG @@ -7393,8 +7385,8 @@ LONG WINAPI CLRVectoredExceptionHandlerShim(PEXCEPTION_POINTERS pExceptionInfo) // WARNING WARNING WARNING WARNING WARNING WARNING WARNING // - // If exceptions (or runtime) have been disabled, then simply return. - if (g_fForbidEnterEE || g_fNoExceptions) + // If runtime have been disabled, then simply return. + if (g_fForbidEnterEE) { return EXCEPTION_CONTINUE_SEARCH; } diff --git a/src/coreclr/vm/i386/excepx86.cpp b/src/coreclr/vm/i386/excepx86.cpp index f0c4dcc8b2a..616cbaea824 100644 --- a/src/coreclr/vm/i386/excepx86.cpp +++ b/src/coreclr/vm/i386/excepx86.cpp @@ -1571,9 +1571,6 @@ EXCEPTION_HANDLER_IMPL(COMPlusFrameHandler) _ASSERTE((pContext == NULL) || ((pContext->ContextFlags & CONTEXT_CONTROL) == CONTEXT_CONTROL)); - if (g_fNoExceptions) - return ExceptionContinueSearch; // No EH during EE shutdown. - // Check if the exception represents a GCStress Marker. If it does, // we shouldnt record its entry in the TLS as such exceptions are // continuable and can confuse the VM to treat them as CSE, diff --git a/src/coreclr/vm/vars.cpp b/src/coreclr/vm/vars.cpp index 00840f91956..6baded4ca7d 100644 --- a/src/coreclr/vm/vars.cpp +++ b/src/coreclr/vm/vars.cpp @@ -193,7 +193,6 @@ GVAL_IMPL(SIZE_T, g_runtimeVirtualSize); Volatile g_fForbidEnterEE = false; bool g_fManagedAttach = false; -bool g_fNoExceptions = false; DWORD g_FinalizerWaiterStatus = 0; diff --git a/src/coreclr/vm/vars.hpp b/src/coreclr/vm/vars.hpp index 03762a24e69..652c8cb7fd5 100644 --- a/src/coreclr/vm/vars.hpp +++ b/src/coreclr/vm/vars.hpp @@ -477,7 +477,6 @@ GVAL_DECL(bool, g_fProcessDetach); GVAL_DECL(bool, g_metadataUpdatesApplied); #endif EXTERN bool g_fManagedAttach; -EXTERN bool g_fNoExceptions; // Indicates whether we're executing shut down as a result of DllMain // (DLL_PROCESS_DETACH). See comments at code:EEShutDown for details. From 3c5a0af402aee0f9e71d38dee9f0c0f1763b8331 Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Mon, 15 Apr 2024 11:47:01 -0700 Subject: [PATCH 069/151] Add workload metadata (#100649) --- src/workloads/workloads.csproj | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/workloads/workloads.csproj b/src/workloads/workloads.csproj index 3db05c0a803..a986fdfdbe6 100644 --- a/src/workloads/workloads.csproj +++ b/src/workloads/workloads.csproj @@ -186,6 +186,15 @@ + + + + + + + + + Date: Mon, 15 Apr 2024 14:08:05 -0700 Subject: [PATCH 070/151] System.Diagnostics.DiagnosticSource (#99583) --- .../src/System.Diagnostics.DiagnosticSource.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj b/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj index 37825100656..013bbb2e0e8 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System.Diagnostics.DiagnosticSource.csproj @@ -6,7 +6,7 @@ $(NoWarn);SA1205 false true - true + false 1 Provides Classes that allow you to decouple code logging rich (unserializable) diagnostics/telemetry (e.g. framework) from code that consumes it (e.g. tools) From 88be910936b852ede1a5e50c2962ddf9ed6f5331 Mon Sep 17 00:00:00 2001 From: Vladimir Sadov Date: Mon, 15 Apr 2024 14:41:14 -0700 Subject: [PATCH 071/151] Always zero-init if object contains pointers (#100265) (#100426) --- src/coreclr/nativeaot/Runtime/gcrhenv.cpp | 6 ++++++ src/libraries/System.Runtime/tests/System/GCTests.cs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp index 379bf5003f9..a0f536b0642 100644 --- a/src/coreclr/nativeaot/Runtime/gcrhenv.cpp +++ b/src/coreclr/nativeaot/Runtime/gcrhenv.cpp @@ -129,6 +129,12 @@ Object* GcAllocInternal(MethodTable *pEEType, uint32_t uFlags, uintptr_t numElem ASSERT(!pThread->IsDoNotTriggerGcSet()); ASSERT(pThread->IsCurrentThreadInCooperativeMode()); + if (pEEType->ContainsPointers()) + { + uFlags |= GC_ALLOC_CONTAINS_REF; + uFlags &= ~GC_ALLOC_ZEROING_OPTIONAL; + } + size_t cbSize = pEEType->get_BaseSize(); if (pEEType->HasComponentSize()) diff --git a/src/libraries/System.Runtime/tests/System/GCTests.cs b/src/libraries/System.Runtime/tests/System/GCTests.cs index d161b3bbdde..14b6e5de406 100644 --- a/src/libraries/System.Runtime/tests/System/GCTests.cs +++ b/src/libraries/System.Runtime/tests/System/GCTests.cs @@ -1100,6 +1100,8 @@ private unsafe static void AllocateArrayPinned_ManagedValueType_CanRoundtripThro byte* pointer = (byte*)Unsafe.AsPointer(ref array[0]); var size = Unsafe.SizeOf>(); + GC.Collect(); + for(int i = 0; i < length; ++i) { int idx = rng.Next(length); From aa7c7ff6cf1d4e8f4b5dd750ded601f379e1c9b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:42:25 -0700 Subject: [PATCH 072/151] [release/8.0-staging] JIT: Fixed incorrect reversed condition for GT (#100372) * Fixed incorrect reversed condition for GT * Added test case * Remove comment * Use JitStressModeNames * Cleaned up test * Update Runtime_92201.csproj --------- Co-authored-by: TIHan --- src/coreclr/jit/gentree.h | 2 +- .../JitBlue/Runtime_92201/Runtime_92201.cs | 174 ++++++++++++++++++ .../Runtime_92201/Runtime_92201.csproj | 13 ++ 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.csproj diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 5a06113fa8d..2720a1e2b18 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -8635,7 +8635,7 @@ struct GenCondition NONE, NONE, SGE, SGT, SLT, SLE, NS, S, NE, EQ, UGE, UGT, ULT, ULE, NC, C, FNEU, FEQU, FGEU, FGTU, FLTU, FLEU, NO, O, - FNE, FEQ, FGE, FGT, FLT, FGT, NP, P + FNE, FEQ, FGE, FGT, FLT, FLE, NP, P }; // clang-format on diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.cs b/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.cs new file mode 100644 index 00000000000..445cc3f915e --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.cs @@ -0,0 +1,174 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using Xunit; + +namespace JIT.HardwareIntrinsics.General._Vector128 +{ + public static partial class Program + { + [Fact] + public static void Test() + { + var test = new VectorBooleanBinaryOpTest__LessThanOrEqualAnySingle(); + + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class VectorBooleanBinaryOpTest__LessThanOrEqualAnySingle + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + + private GCHandle inHandle1; + private GCHandle inHandle2; + + private ulong alignment; + + public DataTable(Single[] inArray1, Single[] inArray2, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf(); + if (!int.IsPow2(alignment) || (alignment > 16) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As(ref inArray2[0]), (uint)sizeOfinArray2); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + } + + private static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + private static readonly int LargestVectorSize = 16; + + private static readonly int Op1ElementCount = Unsafe.SizeOf>() / sizeof(Single); + private static readonly int Op2ElementCount = Unsafe.SizeOf>() / sizeof(Single); + + private static Single[] _data1 = new Single[Op1ElementCount]; + private static Single[] _data2 = new Single[Op2ElementCount]; + + private DataTable _dataTable; + + public VectorBooleanBinaryOpTest__LessThanOrEqualAnySingle() + { + Succeeded = true; + + _data1[0] = 0.168625f; + _data1[1] = 0.5899811f; + _data1[2] = 0.8042229f; + _data1[3] = 0.8173325f; + + _data2[0] = 0.059660614f; + _data2[1] = 0.13952714f; + _data2[2] = 0.23523656f; + _data2[3] = 0.48773053f; + + _dataTable = new DataTable(_data1, _data2, LargestVectorSize); + } + + public bool Succeeded { get; set; } + + [MethodImpl(MethodImplOptions.NoInlining)] + public bool LessThanOrEqualAnyProblem() + { + return Vector128.LessThanOrEqualAny( + Unsafe.Read>(_dataTable.inArray1Ptr), + Unsafe.Read>(_dataTable.inArray2Ptr) + ); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void* GetPtr1() + { + return _dataTable.inArray1Ptr; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void* GetPtr2() + { + return _dataTable.inArray2Ptr; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + public void CheckResult(bool result) + { + ValidateResult(GetPtr1(), GetPtr2(), result); + } + + public void RunBasicScenario_UnsafeRead() + { + var result = Vector128.LessThanOrEqualAny( + Unsafe.Read>(GetPtr1()), + Unsafe.Read>(GetPtr2()) + ); + + CheckResult(result); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ValidateResult(void* op1, void* op2, bool result, [CallerMemberName] string method = "") + { + Single[] inArray1 = new Single[Op1ElementCount]; + Single[] inArray2 = new Single[Op2ElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf>()); + + ValidateResult(inArray1, inArray2, result, method); + } + + private void ValidateResult(Single[] left, Single[] right, bool result, [CallerMemberName] string method = "") + { + bool succeeded = true; + + var expectedResult = false; + + for (var i = 0; i < Op1ElementCount; i++) + { + expectedResult |= (left[i] <= right[i]); + } + + succeeded = (expectedResult == result); + + if (!succeeded) + { + Succeeded = false; + } + } + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.csproj new file mode 100644 index 00000000000..717798d9d4e --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_92201/Runtime_92201.csproj @@ -0,0 +1,13 @@ + + + True + True + + + + + + + + + From f4dc677f6f41404c622cb676cb0b64cdb2926128 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:21:15 -0500 Subject: [PATCH 073/151] Update dependencies from https://github.com/dotnet/emsdk build 20240415.1 (#101047) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.5-servicing.24211.3 -> To Version 8.0.5-servicing.24215.1 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index a0aef486d23..e768edae2ab 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 91df0c87d9c..016d31238f3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - 71359b18c2d83c01a68bf155244a65962a7e8c8e + 6a06b5454e26309fe4ec786857853056c2c541e0 - + https://github.com/dotnet/emsdk - 71359b18c2d83c01a68bf155244a65962a7e8c8e + 6a06b5454e26309fe4ec786857853056c2c541e0 From dce1737abed4e4077e0890b9202dba76aa883059 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:14:47 -0700 Subject: [PATCH 074/151] [release/8.0] Update MicrosoftBuildVersion to latest (#100595) * Update MicrosoftBuildVersion to latest to fix System.Security.Cryptography.XML component governance alert. * Add info to VersionDetails to allow sourcebuild to update the MSBuild dependency * Update SourceBuildPrebuiltBaseline.xml --------- Co-authored-by: Parker Bibus Co-authored-by: Eric StJohn Co-authored-by: Michael Simons --- eng/SourceBuildPrebuiltBaseline.xml | 8 +++++++- eng/Version.Details.xml | 9 +++++++++ eng/Versions.props | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 458b2d756cb..81041ed6ed6 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -28,5 +28,11 @@ + + + + + + - \ No newline at end of file + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 016d31238f3..39a3c00c93f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -410,5 +410,14 @@ https://github.com/dotnet/installer 46a7370763921ded24dcb70c585ee97883c615d4 + + https://github.com/dotnet/msbuild + 195e7f5a3a8e51c37d83cd9e54cb99dc3fc69c22 + + + https://github.com/dotnet/msbuild + 195e7f5a3a8e51c37d83cd9e54cb99dc3fc69c22 + + diff --git a/eng/Versions.props b/eng/Versions.props index d0418b1a22e..3bf003d4987 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -172,7 +172,7 @@ 2.0.3 1.0.4-preview6.19326.1 2.0.5 - 17.3.2 + 17.8.3 $(MicrosoftBuildVersion) $(MicrosoftBuildVersion) $(MicrosoftBuildVersion) From ca4f0fe37455882baa00c75b1ef30a7ff1494457 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:27:42 -0700 Subject: [PATCH 075/151] [release/8.0] Update MicrosoftBuildVersion to latest (#100595) (#101145) * Update MicrosoftBuildVersion to latest to fix System.Security.Cryptography.XML component governance alert. * Add info to VersionDetails to allow sourcebuild to update the MSBuild dependency * Update SourceBuildPrebuiltBaseline.xml --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Parker Bibus Co-authored-by: Eric StJohn Co-authored-by: Michael Simons --- eng/SourceBuildPrebuiltBaseline.xml | 8 +++++++- eng/Version.Details.xml | 9 +++++++++ eng/Versions.props | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 458b2d756cb..81041ed6ed6 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -28,5 +28,11 @@ + + + + + + - \ No newline at end of file + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 91df0c87d9c..fe35dc0997e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -410,5 +410,14 @@ https://github.com/dotnet/installer 46a7370763921ded24dcb70c585ee97883c615d4 + + https://github.com/dotnet/msbuild + 195e7f5a3a8e51c37d83cd9e54cb99dc3fc69c22 + + + https://github.com/dotnet/msbuild + 195e7f5a3a8e51c37d83cd9e54cb99dc3fc69c22 + + diff --git a/eng/Versions.props b/eng/Versions.props index 565f052fc8b..f012e409095 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -172,7 +172,7 @@ 2.0.3 1.0.4-preview6.19326.1 2.0.5 - 17.3.2 + 17.8.3 $(MicrosoftBuildVersion) $(MicrosoftBuildVersion) $(MicrosoftBuildVersion) From b82b0d9e248c62d8433c2ec42a19c24121d7255c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:18:02 +0200 Subject: [PATCH 076/151] [release/8.0-staging] [wasm] Fix AOT publish in paths with space on Windows/Linux (#101109) * Fix * Fix template wbt * Fix config wbt. * Should fix native wbt but does not. * Missing change. * Update src/tasks/AotCompilerTask/MonoAOTCompiler.cs Co-authored-by: Ankit Jain * Double quotes needed, not single. * Space on Linux should work. * Backport https://github.com/dotnet/runtime/pull/94306 --------- Co-authored-by: Ilona Tomkowicz Co-authored-by: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Co-authored-by: Ankit Jain --- .../wasm/Wasm.Build.Tests/Common/BuildEnvironment.cs | 2 +- src/mono/wasm/Wasm.Build.Tests/ConfigSrcTests.cs | 2 +- src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs | 2 +- .../Wasm.Build.Tests/Templates/WasmTemplateTests.cs | 8 ++++---- src/mono/wasm/build/WasmApp.Native.targets | 2 +- src/mono/wasm/build/WasmApp.targets | 12 ++++++------ src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/mono/wasm/Wasm.Build.Tests/Common/BuildEnvironment.cs b/src/mono/wasm/Wasm.Build.Tests/Common/BuildEnvironment.cs index 21feaea66f2..0674991bc1d 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Common/BuildEnvironment.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Common/BuildEnvironment.cs @@ -30,7 +30,7 @@ public class BuildEnvironment public static readonly string RelativeTestAssetsPath = @"..\testassets\"; public static readonly string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets"); public static readonly string TestDataPath = Path.Combine(AppContext.BaseDirectory, "data"); - public static readonly string TmpPath = Path.Combine(AppContext.BaseDirectory, "wbt"); + public static readonly string TmpPath = Path.Combine(AppContext.BaseDirectory, "wbt artifacts"); public static readonly string DefaultRuntimeIdentifier = #if TARGET_WASI diff --git a/src/mono/wasm/Wasm.Build.Tests/ConfigSrcTests.cs b/src/mono/wasm/Wasm.Build.Tests/ConfigSrcTests.cs index ac71dafe0eb..329ecbe0b49 100644 --- a/src/mono/wasm/Wasm.Build.Tests/ConfigSrcTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/ConfigSrcTests.cs @@ -32,6 +32,6 @@ public void ConfigSrcAbsolutePath(BuildArgs buildArgs, RunHost host, string id) string bundleDir = Path.Combine(binDir, "AppBundle"); string configSrc = Path.GetFullPath(Path.Combine(bundleDir, "_framework", "blazor.boot.json")); - RunAndTestWasmApp(buildArgs, expectedExitCode: 42, host: host, id: id, extraXHarnessMonoArgs: $"--config-src={configSrc}"); + RunAndTestWasmApp(buildArgs, expectedExitCode: 42, host: host, id: id, extraXHarnessMonoArgs: $"--config-src=\"{configSrc}\""); } } diff --git a/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs b/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs index 0ea7aaf7bfa..3d30a5c8411 100644 --- a/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs @@ -70,7 +70,7 @@ public void IntermediateBitcodeToObjectFilesAreNotLLVMIR(BuildArgs buildArgs, st { string printFileTypeTarget = @" - diff --git a/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTests.cs b/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTests.cs index 08488d487a6..821fd528e22 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Templates/WasmTemplateTests.cs @@ -277,7 +277,7 @@ private async Task BrowserRunTwiceWithAndThenWithoutBuildAsync(string config, st .WithWorkingDirectory(workingDir); await using var runner = new BrowserRunner(_testOutput); - var page = await runner.RunAsync(runCommand, $"run --no-silent -c {config} --project {projectFile} --forward-console"); + var page = await runner.RunAsync(runCommand, $"run --no-silent -c {config} --project \"{projectFile}\" --forward-console"); await runner.WaitForExitMessageAsync(TimeSpan.FromMinutes(2)); Assert.Contains("Hello, Browser!", string.Join(Environment.NewLine, runner.OutputLines)); } @@ -287,7 +287,7 @@ private async Task BrowserRunTwiceWithAndThenWithoutBuildAsync(string config, st .WithWorkingDirectory(workingDir); await using var runner = new BrowserRunner(_testOutput); - var page = await runner.RunAsync(runCommand, $"run --no-silent -c {config} --no-build --project {projectFile} --forward-console"); + var page = await runner.RunAsync(runCommand, $"run --no-silent -c {config} --no-build --project \"{projectFile}\" --forward-console"); await runner.WaitForExitMessageAsync(TimeSpan.FromMinutes(2)); Assert.Contains("Hello, Browser!", string.Join(Environment.NewLine, runner.OutputLines)); } @@ -307,7 +307,7 @@ private Task ConsoleRunWithAndThenWithoutBuildAsync(string config, string extraP string workingDir = runOutsideProjectDirectory ? BuildEnvironment.TmpPath : _projectDir!; { - string runArgs = $"run --no-silent -c {config} --project {projectFile}"; + string runArgs = $"run --no-silent -c {config} --project \"{projectFile}\""; runArgs += " x y z"; using var cmd = new RunCommand(s_buildEnv, _testOutput, label: id) .WithWorkingDirectory(workingDir) @@ -323,7 +323,7 @@ private Task ConsoleRunWithAndThenWithoutBuildAsync(string config, string extraP { // Run with --no-build - string runArgs = $"run --no-silent -c {config} --project {projectFile} --no-build"; + string runArgs = $"run --no-silent -c {config} --project \"{projectFile}\" --no-build"; runArgs += " x y z"; using var cmd = new RunCommand(s_buildEnv, _testOutput, label: id) .WithWorkingDirectory(workingDir); diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 73e5720abb0..52b2988f8dd 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -415,7 +415,7 @@ - - - + + <_HasDotnetWasm Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.native.wasm'">true <_HasDotnetJsWorker Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.native.worker.js'">true @@ -415,7 +415,7 @@ Outputs="$(WasmAppDir)\.stamp" Condition="'$(WasmGenerateAppBundle)' == 'true'" DependsOnTargets="_WasmGenerateRuntimeConfig;_GetWasmGenerateAppBundleDependencies"> - + <_WasmAppIncludeThreadsWorker Condition="'$(WasmEnableThreads)' == 'true' or '$(MonoWasmBuildVariant)' == 'multithread'">true @@ -473,7 +473,7 @@ - + - + diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 9e52322770b..0a761b44a4c 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -951,7 +951,7 @@ private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, st if (isDedup) { foreach (var aItem in _assembliesToCompile!) - processArgs.Add(aItem.ItemSpec); + processArgs.Add($"\"{aItem.ItemSpec}\""); } else { From 4cbd412768d46c3fe64421c5e3adc30b630f5308 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 09:58:21 -0500 Subject: [PATCH 077/151] Update dependencies from https://github.com/dotnet/source-build-externals build 20240416.1 (#101181) Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24175.3 -> To Version 8.0.0-alpha.1.24216.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 39a3c00c93f..04d355fd7b9 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 79827eed138fd2575a8b24820b4f385ee4ffb6e6 - + https://github.com/dotnet/source-build-externals - 300e99190e6ae1983681694dbdd5f75f0c692081 + 908177a58a41532b3302c17f1e1a8cf1c1234545 From 82025822b6e74bd7f19c5d73ad04ba0e104e6b9a Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 17 Apr 2024 13:00:17 -0700 Subject: [PATCH 078/151] Update queues. (#101094) --- eng/testing/performance/performance-setup.ps1 | 14 +++++++------- eng/testing/performance/performance-setup.sh | 12 +++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index c18445bff8a..2ce32b70c59 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -14,7 +14,7 @@ Param( [string] $Kind="micro", [switch] $LLVM, [switch] $MonoInterpreter, - [switch] $MonoAOT, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, [string] $MonoDotnet="", @@ -47,14 +47,14 @@ $Queue = "" if ($Internal) { switch ($LogicalMachine) { - "perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - "perftiger_crossgen" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - "perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf" } - "perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf" } + "perftiger" { $Queue = "Windows.11.Amd64.Tiger.Perf" } + "perftiger_crossgen" { $Queue = "Windows.11.Amd64.Tiger.Perf" } + "perfowl" { $Queue = "Windows.11.Amd64.Owl.Perf" } + "perfsurf" { $Queue = "Windows.11.Arm64.Surf.Perf" } "perfpixel4a" { $Queue = "Windows.11.Amd64.Pixel.Perf" } "perfampere" { $Queue = "Windows.Server.Arm64.Perf" } "cloudvm" { $Queue = "Windows.10.Amd64" } - Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } + Default { $Queue = "Windows.11.Amd64.Tiger.Perf" } } $PerfLabArguments = "--upload-to-perflab-container" $ExtraBenchmarkDotNetArguments = "" @@ -124,7 +124,7 @@ if ($UseLocalCommitTime) { if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" - + robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh index 4853f0e6fb3..eb49d3ea13d 100755 --- a/eng/testing/performance/performance-setup.sh +++ b/eng/testing/performance/performance-setup.sh @@ -264,7 +264,7 @@ benchmark_directory=$payload_directory/BenchmarkDotNet workitem_directory=$source_directory/workitem extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" perflab_arguments= -queue=Ubuntu.1804.Amd64.Open +queue=Ubuntu.2204.Amd64.Open creator=$BUILD_DEFINITIONNAME helix_source_prefix="pr" @@ -277,14 +277,16 @@ if [[ "$internal" == true ]]; then if [[ "$logical_machine" == "perfiphone12mini" ]]; then queue=OSX.13.Amd64.Iphone.Perf elif [[ "$logical_machine" == "perfampere" ]]; then - queue=Ubuntu.2004.Arm64.Perf + queue=Ubuntu.2204.Arm64.Perf + elif [[ "$logical_machine" == "perfviper" ]]; then + queue=Ubuntu.2204.Amd64.Viper.Perf elif [[ "$logical_machine" == "cloudvm" ]]; then - queue=Ubuntu.1804.Amd64 + queue=Ubuntu.2204.Amd64 elif [[ "$architecture" == "arm64" ]]; then queue=Ubuntu.1804.Arm64.Perf else if [[ "$logical_machine" == "perfowl" ]]; then - queue=Ubuntu.1804.Amd64.Owl.Perf + queue=Ubuntu.2204.Amd64.Owl.Perf elif [[ "$logical_machine" == "perftiger_crossgen" ]]; then queue=Ubuntu.1804.Amd64.Tiger.Perf else @@ -299,7 +301,7 @@ else if [[ "$architecture" == "arm64" ]]; then queue=ubuntu.1804.armarch.open else - queue=Ubuntu.1804.Amd64.Open + queue=Ubuntu.2204.Amd64.Open fi if [[ "$alpine" == "true" ]]; then From d3da39dd703c6af8d700ad46beba3084302825ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:41:22 -0700 Subject: [PATCH 079/151] [release/8.0-staging] fix SendTo with SocketAsyncEventArgs (#99695) * fix SendTo with SocketAsyncEventArgs * feedback --------- Co-authored-by: wfurt --- .../src/System/Net/Sockets/Socket.Tasks.cs | 12 ++++- .../src/System/Net/Sockets/Socket.cs | 20 +++++--- .../Net/Sockets/SocketAsyncEventArgs.cs | 7 ++- .../tests/FunctionalTests/SendTo.cs | 29 ++++++++++++ .../SocketAsyncEventArgsTest.cs | 47 +++++++++++++++++++ 5 files changed, 106 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs index e1891bef916..3ba24e90cf1 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs @@ -677,7 +677,6 @@ public ValueTask SendToAsync(ReadOnlyMemory buffer, SocketFlags socke Debug.Assert(saea.BufferList == null); saea.SetBuffer(MemoryMarshal.AsMemory(buffer)); saea.SocketFlags = socketFlags; - saea._socketAddress = null; saea.RemoteEndPoint = remoteEP; saea.WrapExceptionsForNetworkStream = false; return saea.SendToAsync(this, cancellationToken); @@ -709,8 +708,17 @@ public ValueTask SendToAsync(ReadOnlyMemory buffer, SocketFlags socke saea.SetBuffer(MemoryMarshal.AsMemory(buffer)); saea.SocketFlags = socketFlags; saea._socketAddress = socketAddress; + saea.RemoteEndPoint = null; saea.WrapExceptionsForNetworkStream = false; - return saea.SendToAsync(this, cancellationToken); + try + { + return saea.SendToAsync(this, cancellationToken); + } + finally + { + // detach user provided SA so we do not accidentally stomp on it later. + saea._socketAddress = null; + } } /// diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs index 414b0baa86e..2a88a70f594 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs @@ -3095,14 +3095,22 @@ private bool SendToAsync(SocketAsyncEventArgs e, CancellationToken cancellationT ArgumentNullException.ThrowIfNull(e); EndPoint? endPointSnapshot = e.RemoteEndPoint; - if (e._socketAddress == null) + + // RemoteEndPoint should be set unless somebody used SendTo with their own SA. + // In that case RemoteEndPoint will be null and we take provided SA as given. + if (endPointSnapshot == null && e._socketAddress == null) { - if (endPointSnapshot == null) - { - throw new ArgumentException(SR.Format(SR.InvalidNullArgument, "e.RemoteEndPoint"), nameof(e)); - } + throw new ArgumentException(SR.Format(SR.InvalidNullArgument, "e.RemoteEndPoint"), nameof(e)); + } - // Prepare SocketAddress + if (e._socketAddress != null && endPointSnapshot is IPEndPoint ipep && e._socketAddress.Family == endPointSnapshot?.AddressFamily) + { + // we have matching SocketAddress. Since this is only used internally, it is ok to overwrite it without + ipep.Serialize(e._socketAddress.Buffer.Span); + } + else if (endPointSnapshot != null) + { + // Prepare new SocketAddress e._socketAddress = Serialize(ref endPointSnapshot); } diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs index e94d862571a..78dd22e5eda 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs @@ -923,7 +923,12 @@ internal void FinishOperationSyncSuccess(int bytesTransferred, SocketFlags flags case SocketAsyncOperation.ReceiveFrom: // Deal with incoming address. UpdateReceivedSocketAddress(_socketAddress!); - if (_remoteEndPoint != null && !SocketAddressExtensions.Equals(_socketAddress!, _remoteEndPoint)) + if (_remoteEndPoint == null) + { + // detach user provided SA as it was updated in place. + _socketAddress = null; + } + else if (!SocketAddressExtensions.Equals(_socketAddress!, _remoteEndPoint)) { try { diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs index bf0ad146588..7a3c33b64bf 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs @@ -173,6 +173,35 @@ public void SendToAsync_NullAsyncEventArgs_Throws_ArgumentNullException() public sealed class SendTo_Task : SendTo { public SendTo_Task(ITestOutputHelper output) : base(output) { } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SendTo_DifferentEP_Success(bool ipv4) + { + IPAddress address = ipv4 ? IPAddress.Loopback : IPAddress.IPv6Loopback; + IPEndPoint remoteEp = new IPEndPoint(address, 0); + + using Socket receiver1 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + using Socket receiver2 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + using Socket sender = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + + receiver1.BindToAnonymousPort(address); + receiver2.BindToAnonymousPort(address); + + byte[] sendBuffer = new byte[32]; + var receiveInternalBuffer = new byte[sendBuffer.Length]; + ArraySegment receiveBuffer = new ArraySegment(receiveInternalBuffer, 0, receiveInternalBuffer.Length); + + + await sender.SendToAsync(sendBuffer, SocketFlags.None, receiver1.LocalEndPoint); + SocketReceiveFromResult result = await ReceiveFromAsync(receiver1, receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); + Assert.Equal(sendBuffer.Length, result.ReceivedBytes); + + await sender.SendToAsync(sendBuffer, SocketFlags.None, receiver2.LocalEndPoint); + result = await ReceiveFromAsync(receiver2, receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); + Assert.Equal(sendBuffer.Length, result.ReceivedBytes); + } } public sealed class SendTo_CancellableTask : SendTo diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs index ded34276f32..3d865cb8645 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs @@ -895,5 +895,52 @@ void CreateSocketAsyncEventArgs() // separated out so that JIT doesn't extend li return cwt.Count() == 0; // validate that the cwt becomes empty }, 30_000)); } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task SendTo_DifferentEP_Success(bool ipv4) + { + IPAddress address = ipv4 ? IPAddress.Loopback : IPAddress.IPv6Loopback; + IPEndPoint remoteEp = new IPEndPoint(address, 0); + + using Socket receiver1 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + using Socket receiver2 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + using Socket sender = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); + + receiver1.BindToAnonymousPort(address); + receiver2.BindToAnonymousPort(address); + + byte[] sendBuffer = new byte[32]; + var receiveInternalBuffer = new byte[sendBuffer.Length]; + ArraySegment receiveBuffer = new ArraySegment(receiveInternalBuffer, 0, receiveInternalBuffer.Length); + + using SocketAsyncEventArgs saea = new SocketAsyncEventArgs(); + ManualResetEventSlim mres = new ManualResetEventSlim(false); + + saea.SetBuffer(sendBuffer); + saea.RemoteEndPoint = receiver1.LocalEndPoint; + saea.Completed += delegate { mres.Set(); }; + if (sender.SendToAsync(saea)) + { + // did not finish synchronously. + mres.Wait(); + } + + SocketReceiveFromResult result = await receiver1.ReceiveFromAsync(receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); + Assert.Equal(sendBuffer.Length, result.ReceivedBytes); + mres.Reset(); + + + saea.RemoteEndPoint = receiver2.LocalEndPoint; + if (sender.SendToAsync(saea)) + { + // did not finish synchronously. + mres.Wait(); + } + + result = await receiver2.ReceiveFromAsync(receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); + Assert.Equal(sendBuffer.Length, result.ReceivedBytes); + } } } From 47afa791f1d8a8e9c380f1fa84dc7d94624bec1a Mon Sep 17 00:00:00 2001 From: Matous Kozak <55735845+matouskozak@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:11:54 +0200 Subject: [PATCH 080/151] [release/8.0][mono][HybridGlobalization] Fix ShortDatePattern year format to be "yyyy" (#100810) * fix short date year format on apple hybrid glob. * add test for ShortDatePattern for iOS hybrid globalization --- .../DateTimeFormatInfoShortDatePattern.cs | 15 ++++++++++++++- .../src/System/Globalization/CalendarData.iOS.cs | 9 +++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Globalization/tests/DateTimeFormatInfo/DateTimeFormatInfoShortDatePattern.cs b/src/libraries/System.Globalization/tests/DateTimeFormatInfo/DateTimeFormatInfoShortDatePattern.cs index 815533bd1d8..7d4b146b9de 100644 --- a/src/libraries/System.Globalization/tests/DateTimeFormatInfo/DateTimeFormatInfoShortDatePattern.cs +++ b/src/libraries/System.Globalization/tests/DateTimeFormatInfo/DateTimeFormatInfoShortDatePattern.cs @@ -8,6 +8,13 @@ namespace System.Globalization.Tests { public class DateTimeFormatInfoShortDatePattern { + public static IEnumerable ShortDatePattern_Get_TestData() + { + yield return new object[] { DateTimeFormatInfo.InvariantInfo, "MM/dd/yyyy", "invariant" }; + yield return new object[] { new CultureInfo("en-US").DateTimeFormat, "M/d/yyyy", "en-US" }; + yield return new object[] { new CultureInfo("fr-FR").DateTimeFormat, "dd/MM/yyyy", "fr-FR" }; + } + public static IEnumerable ShortDatePattern_Get_TestData_HybridGlobalization() { // see the comments on the right to check the non-Hybrid result, if it differs @@ -131,7 +138,6 @@ public static IEnumerable ShortDatePattern_Get_TestData_HybridGlobaliz yield return new object[] { new CultureInfo("en-ZA").DateTimeFormat, "yyyy/MM/dd" }; yield return new object[] { new CultureInfo("en-ZM").DateTimeFormat, "dd/MM/yyyy" }; yield return new object[] { new CultureInfo("en-ZW").DateTimeFormat, "d/M/yyyy" }; - yield return new object[] { new CultureInfo("en-US").DateTimeFormat, "M/d/yyyy" }; yield return new object[] { new CultureInfo("es-419").DateTimeFormat, "d/M/yyyy" }; yield return new object[] { new CultureInfo("es-ES").DateTimeFormat, "d/M/yyyy" }; yield return new object[] { new CultureInfo("es-MX").DateTimeFormat, "dd/MM/yyyy" }; @@ -200,6 +206,13 @@ public static IEnumerable ShortDatePattern_Get_TestData_HybridGlobaliz yield return new object[] { new CultureInfo("zh-TW").DateTimeFormat, "yyyy/M/d" }; } + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsHybridGlobalizationOnOSX))] + [MemberData(nameof(ShortDatePattern_Get_TestData))] + public void ShortDatePattern_Get_ReturnsExpected(DateTimeFormatInfo format, string expected, string cultureName) + { + Assert.True(expected == format.ShortDatePattern, $"Failed for culture: {cultureName}. Expected: {expected}, Actual: {format.ShortDatePattern}"); + } + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsHybridGlobalizationOnBrowser))] [MemberData(nameof(ShortDatePattern_Get_TestData_HybridGlobalization))] public void ShortDatePattern_Get_ReturnsExpected_HybridGlobalization(DateTimeFormatInfo format, string expected) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.iOS.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.iOS.cs index 46bfb3d481c..98925284412 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.iOS.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.iOS.cs @@ -18,6 +18,15 @@ private bool LoadCalendarDataFromNative(string localeName, CalendarId calendarId sNativeName = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.NativeName); sMonthDay = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.MonthDay); saShortDates = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.ShortDates).Split("||"); + // Handle ShortDatePattern to have "yyyy" year format + List shortDatePatternList = new List(saShortDates); + for (int i = 0; i < shortDatePatternList.Count; i++) + { + shortDatePatternList[i] = NormalizeDatePattern(shortDatePatternList[i]); + } + FixDefaultShortDatePattern(shortDatePatternList); + saShortDates = shortDatePatternList.ToArray(); + saLongDates = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.LongDates).Split("||"); saYearMonths = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.YearMonths).Split("||"); saDayNames = GetCalendarInfoNative(localeName, calendarId, CalendarDataType.DayNames).Split("||"); From d597a37331b8b493129ebea628dbf1256b6c1770 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 19 Apr 2024 09:59:38 -0700 Subject: [PATCH 081/151] [release/8.0-staging] Remove Win7 Helix queues (#100985) Co-authored-by: Jeremy Koritzinsky --- eng/pipelines/libraries/helix-queues-setup.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index be98c70f161..8acabe6ec38 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -152,7 +152,6 @@ jobs: - ${{ if notIn(parameters.jobParameters.framework, 'net48') }}: # mono outerloop - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - - Windows.7.Amd64.Open - Windows.11.Amd64.Client.Open # libraries on coreclr (outerloop and innerloop), or libraries on mono innerloop - ${{ if or(ne(parameters.jobParameters.testScope, 'outerloop'), ne(parameters.jobParameters.runtimeFlavor, 'mono')) }}: @@ -161,7 +160,6 @@ jobs: - Windows.Amd64.Server2022.Open - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - Windows.Amd64.Server2022.Open - - Windows.7.Amd64.Open # .NETFramework - ${{ if eq(parameters.jobParameters.framework, 'net48') }}: From 6c1f1c3e449df96024334adc8c67af2c14450924 Mon Sep 17 00:00:00 2001 From: Aman Khalid Date: Fri, 19 Apr 2024 18:38:00 +0000 Subject: [PATCH 082/151] Merge #91634 into release/8.0-staging (#100994) --- eng/empty.csproj | 17 ----------------- .../runtimes/send-to-helix-inner-step.yml | 19 ++----------------- .../templates/runtimes/send-to-helix-step.yml | 1 - eng/pipelines/coreclr/templates/build-job.yml | 17 ----------------- eng/pipelines/mono/templates/build-job.yml | 19 ------------------- 5 files changed, 2 insertions(+), 71 deletions(-) delete mode 100644 eng/empty.csproj diff --git a/eng/empty.csproj b/eng/empty.csproj deleted file mode 100644 index 55eefdef1ad..00000000000 --- a/eng/empty.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - $(NetCoreAppToolCurrent) - - - - - diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml index bd147de7e77..12e3f68baaf 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml @@ -1,6 +1,5 @@ parameters: osGroup: '' - restoreParams: '' sendParams: '' condition: '' displayName: '' @@ -9,28 +8,14 @@ parameters: steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - # TODO: Remove and consolidate this when we move to arcade via init-tools.cmd. - - powershell: $(Build.SourcesDirectory)\eng\common\build.ps1 -ci -warnaserror 0 ${{ parameters.restoreParams }} - displayName: Restore blob feed tasks (Windows) - condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.restoreParams, '')) }}) - - - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 -ci -warnaserror 0 ${{ parameters.sendParams }} + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 --restore -ci -warnaserror 0 ${{ parameters.sendParams }} displayName: ${{ parameters.displayName }} (Windows) condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.sendParams, '')) }}) env: ${{ parameters.environment }} continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} - ${{ if ne(parameters.osGroup, 'windows') }}: - # TODO: Remove and consolidate this when we move to arcade via init-tools.sh. - - script: $(Build.SourcesDirectory)/eng/common/build.sh --ci --warnaserror false ${{ parameters.restoreParams }} - displayName: Restore blob feed tasks (Unix) - condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.restoreParams, '')) }}) - ${{ if eq(parameters.osGroup, 'freebsd') }}: - env: - # Arcade uses this SDK instead of trying to restore one. - DotNetCoreSdkDir: /usr/local/dotnet - - - script: $(Build.SourcesDirectory)/eng/common/msbuild.sh --ci --warnaserror false ${{ parameters.sendParams }} + - script: $(Build.SourcesDirectory)/eng/common/msbuild.sh --restore --ci --warnaserror false ${{ parameters.sendParams }} displayName: ${{ parameters.displayName }} (Unix) condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.sendParams, '')) }}) env: ${{ parameters.environment }} diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index 37b455b19d8..97d565b3a4e 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -38,7 +38,6 @@ steps: - template: send-to-helix-inner-step.yml parameters: osGroup: ${{ parameters.osGroup }} - restoreParams: /p:DotNetPublishToBlobFeed=true -restore -projects $(Build.SourcesDirectory)$(dir)eng$(dir)empty.csproj sendParams: ${{ parameters.helixProjectArguments }} ${{ parameters.msbuildParallelism }} /bl:$(Build.SourcesDirectory)/artifacts/log/SendToHelix.binlog /p:TargetArchitecture=${{ parameters.archType }} /p:TargetOS=${{ parameters.osGroup }} /p:TargetOSSubgroup=${{ parameters.osSubgroup }} /p:Configuration=${{ parameters.buildConfig }} condition: and(succeeded(), ${{ parameters.condition }}) shouldContinueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 365c1432aa4..2e128ea54e1 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -70,15 +70,6 @@ jobs: value: '' - name: publishLogsArtifactPrefix value: 'BuildLogs_CoreCLR' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - # Variables used to publish packages to blob feed - - name: dotnetfeedUrl - value: https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json - - name: dotnetfeedPAT - value: $(dotnetfeed-storage-access-key-1) - # Variables used by arcade to gather asset manifests - - name: _DotNetPublishToBlobFeed - value: true - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: AzureDevOps-Artifact-Feeds-Pats - name: officialBuildIdArg @@ -298,14 +289,6 @@ jobs: artifactName: $(nativeTestArtifactName) displayName: 'native test components' - # Get key vault secrets for publishing - - ${{ if and(ne(variables['System.TeamProject'], 'public'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'DotNet-Engineering-Services_KeyVault' - KeyVaultName: EngKeyVault - SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat' - # Save packages using the prepare-signed-artifacts format. - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.pgoType, '')) }}: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index b1ac9c1cc65..845078804fb 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -171,25 +171,6 @@ jobs: - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) displayName: Build nupkg - # Publish official build - - ${{ if eq(parameters.publishToBlobFeed, 'true') }}: - - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/eng/common/build.sh --ci --restore --publish --configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:Configuration=$(_BuildConfig) /p:TargetArchitecture=$(archType) /p:TargetOS=$(osGroup) /p:OSIdentifier=$(osGroup)$(osSubgroup) /bl:"$(Build.SourcesDirectory)/artifacts/log/publish-pkgs.binlog" --projects $(Build.SourcesDirectory)/eng/empty.csproj - displayName: Publish packages to blob feed - env: - # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed - NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages - ${{ if eq(parameters.osGroup, 'freebsd') }}: - # Arcade uses this SDK instead of trying to restore one. - DotNetCoreSdkDir: /usr/local/dotnet - - ${{ if eq(parameters.osGroup, 'windows') }}: - # TODO: pass publish feed url and access token in from the internal pipeline - - powershell: eng\common\build.ps1 -ci -restore -publish -configuration $(_BuildConfig) /p:DotNetPublishUsingPipelines=true /p:DotNetPublishToBlobFeed=true /p:DotNetPublishBlobFeedUrl=$(dotnetfeedUrl) /p:DotNetPublishBlobFeedKey=$(dotnetfeedPAT) /p:Configuration=$(_BuildConfig) /p:TargetArchitecture=$(archType) /p:TargetOS=$(osGroup) /p:OSIdentifier=$(osGroup)$(osSubgroup) /bl:"$(Build.SourcesDirectory)\artifacts\log\publish-pkgs.binlog" -projects $(Build.SourcesDirectory)\eng\empty.csproj - displayName: Publish packages to blob feed - env: - # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed - NUGET_PACKAGES: $(Build.SourcesDirectory)\.packages - # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs From 0b84eae7fb5b2d577cdcec32446a9799de562494 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 12:29:52 +0200 Subject: [PATCH 083/151] [release/8.0] Avoid rooting X509Certificate2 in SslSessionCache (#101144) * Avoid rooting X509Certificate2 in SslSessionCache * Update src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs Co-authored-by: campersau --------- Co-authored-by: Radek Zikmund Co-authored-by: Radek Zikmund <32671551+rzikm@users.noreply.github.com> Co-authored-by: campersau --- .../Common/src/Interop/Windows/SspiCli/SecuritySafeHandles.cs | 3 +-- .../src/System/Net/CertificateValidationPal.Windows.cs | 2 +- .../src/System/Net/Security/SslStreamPal.Windows.cs | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/SspiCli/SecuritySafeHandles.cs b/src/libraries/Common/src/Interop/Windows/SspiCli/SecuritySafeHandles.cs index 91952869883..272232f414a 100644 --- a/src/libraries/Common/src/Interop/Windows/SspiCli/SecuritySafeHandles.cs +++ b/src/libraries/Common/src/Interop/Windows/SspiCli/SecuritySafeHandles.cs @@ -312,13 +312,12 @@ internal sealed class SafeFreeCredential_SECURITY : SafeFreeCredentials { #pragma warning disable 0649 // This is used only by SslStream but it is included elsewhere - public X509Certificate? LocalCertificate; + public bool HasLocalCertificate; #pragma warning restore 0649 public SafeFreeCredential_SECURITY() : base() { } protected override bool ReleaseHandle() { - LocalCertificate?.Dispose(); return Interop.SspiCli.FreeCredentialsHandle(ref _handle) == 0; } } diff --git a/src/libraries/System.Net.Security/src/System/Net/CertificateValidationPal.Windows.cs b/src/libraries/System.Net.Security/src/System/Net/CertificateValidationPal.Windows.cs index f0e58802dbd..083c6616610 100644 --- a/src/libraries/System.Net.Security/src/System/Net/CertificateValidationPal.Windows.cs +++ b/src/libraries/System.Net.Security/src/System/Net/CertificateValidationPal.Windows.cs @@ -105,7 +105,7 @@ internal static bool IsLocalCertificateUsed(SafeFreeCredentials? _credentialsHan // This is TLS Resumed session. Windows can fail to query the local cert bellow. // Instead, we will determine the usage form used credentials. SafeFreeCredential_SECURITY creds = (SafeFreeCredential_SECURITY)_credentialsHandle!; - return creds.LocalCertificate != null; + return creds.HasLocalCertificate; } SafeFreeCertContext? localContext = null; diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs index 45e77f49b6e..3ee2c8ac561 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs @@ -231,8 +231,7 @@ public static SafeFreeCredentials AcquireCredentialsHandle(SslAuthenticationOpti if (newCredentialsRequested && sslAuthenticationOptions.CertificateContext != null) { SafeFreeCredential_SECURITY handle = (SafeFreeCredential_SECURITY)cred; - // We need to create copy to avoid Disposal issue. - handle.LocalCertificate = new X509Certificate2(sslAuthenticationOptions.CertificateContext.TargetCertificate); + handle.HasLocalCertificate = true; } return cred; From 535dd08e372abebbc3c92c892af0939c1aa63676 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:43:29 -0700 Subject: [PATCH 084/151] crashinfo.cpp/crasinfounix.cpp: use off_t instead of off64_t (#101272) Co-authored-by: Antoine Martin --- src/coreclr/debug/createdump/crashinfo.cpp | 2 +- src/coreclr/debug/createdump/crashinfounix.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/debug/createdump/crashinfo.cpp b/src/coreclr/debug/createdump/crashinfo.cpp index 996f3a81935..d1d1b0ea021 100644 --- a/src/coreclr/debug/createdump/crashinfo.cpp +++ b/src/coreclr/debug/createdump/crashinfo.cpp @@ -803,7 +803,7 @@ CrashInfo::PageMappedToPhysicalMemory(uint64_t start) } uint64_t pagemapOffset = (start / PAGE_SIZE) * sizeof(uint64_t); - uint64_t seekResult = lseek64(m_fdPagemap, (off64_t) pagemapOffset, SEEK_SET); + uint64_t seekResult = lseek(m_fdPagemap, (off_t) pagemapOffset, SEEK_SET); if (seekResult != pagemapOffset) { int seekErrno = errno; diff --git a/src/coreclr/debug/createdump/crashinfounix.cpp b/src/coreclr/debug/createdump/crashinfounix.cpp index 24b975e3d65..2f4ea079de3 100644 --- a/src/coreclr/debug/createdump/crashinfounix.cpp +++ b/src/coreclr/debug/createdump/crashinfounix.cpp @@ -516,7 +516,7 @@ CrashInfo::ReadProcessMemory(void* address, void* buffer, size_t size, size_t* r // performance optimization. m_canUseProcVmReadSyscall = false; assert(m_fdMem != -1); - *read = pread64(m_fdMem, buffer, size, (off64_t)address); + *read = pread(m_fdMem, buffer, size, (off_t)address); } if (*read == (size_t)-1) From def0e85f3c050cc150f2261bdd9d82518362bc00 Mon Sep 17 00:00:00 2001 From: Steve Harter Date: Tue, 23 Apr 2024 14:03:38 -0500 Subject: [PATCH 085/151] TypeDescriptor threading fixes (#101305) --- .../System/ComponentModel/TypeDescriptor.cs | 62 +++-- .../tests/TypeDescriptorTests.cs | 247 ++++++++++++++++++ 2 files changed, 286 insertions(+), 23 deletions(-) diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs index 2efd53e6223..948e3f4b386 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs @@ -28,7 +28,12 @@ public sealed class TypeDescriptor // class load anyway. private static readonly WeakHashtable s_providerTable = new WeakHashtable(); // mapping of type or object hash to a provider list private static readonly Hashtable s_providerTypeTable = new Hashtable(); // A direct mapping from type to provider. - private static readonly Hashtable s_defaultProviders = new Hashtable(); // A table of type -> default provider to track DefaultTypeDescriptionProviderAttributes. + + private static readonly Hashtable s_defaultProviderInitialized = new Hashtable(); // A table of type -> object to track DefaultTypeDescriptionProviderAttributes. + // A value of `null` indicates initialization is in progress. + // A value of s_initializedDefaultProvider indicates the provider is initialized. + private static readonly object s_initializedDefaultProvider = new object(); + private static WeakHashtable? s_associationTable; private static int s_metadataVersion; // a version stamp for our metadata. Used by property descriptors to know when to rebuild attributes. @@ -75,8 +80,6 @@ public sealed class TypeDescriptor Guid.NewGuid() // events }; - private static readonly object s_internalSyncObject = new object(); - private TypeDescriptor() { } @@ -262,32 +265,43 @@ public static void AddProviderTransparent(TypeDescriptionProvider provider, obje /// private static void CheckDefaultProvider(Type type) { - if (s_defaultProviders.ContainsKey(type)) + if (s_defaultProviderInitialized[type] == s_initializedDefaultProvider) { return; } - lock (s_internalSyncObject) + // Lock on s_providerTable even though s_providerTable is not modified here. + // Using a single lock prevents deadlocks since other methods that call into or are called + // by this method also lock on s_providerTable and the ordering of the locks may be different. + lock (s_providerTable) { - if (s_defaultProviders.ContainsKey(type)) - { - return; - } + AddDefaultProvider(type); + } + } - // Immediately clear this. If we find a default provider - // and it starts messing around with type information, - // this could infinitely recurse. - s_defaultProviders[type] = null; + /// + /// Add the default provider, if it exists. + /// For threading, this is always called under a 'lock (s_providerTable)'. + /// + private static void AddDefaultProvider(Type type) + { + bool providerAdded = false; + + if (s_defaultProviderInitialized.ContainsKey(type)) + { + // Either another thread finished initializing for this type, or we are recursing on the same thread. + return; } - // Always use core reflection when checking for - // the default provider attribute. If there is a - // provider, we probably don't want to build up our - // own cache state against the type. There shouldn't be - // more than one of these, but walk anyway. Walk in - // reverse order so that the most derived takes precidence. + // Immediately set this to null to indicate we are in progress setting the default provider for a type. + // This prevents re-entrance to this method. + s_defaultProviderInitialized[type] = null; + + // Always use core reflection when checking for the default provider attribute. + // If there is a provider, we probably don't want to build up our own cache state against the type. + // There shouldn't be more than one of these, but walk anyway. + // Walk in reverse order so that the most derived takes precedence. object[] attrs = type.GetCustomAttributes(typeof(TypeDescriptionProviderAttribute), false); - bool providerAdded = false; for (int idx = attrs.Length - 1; idx >= 0; idx--) { TypeDescriptionProviderAttribute pa = (TypeDescriptionProviderAttribute)attrs[idx]; @@ -306,17 +320,19 @@ private static void CheckDefaultProvider(Type type) Type? baseType = type.BaseType; if (baseType != null && baseType != type) { - CheckDefaultProvider(baseType); + AddDefaultProvider(baseType); } } + + s_defaultProviderInitialized[type] = s_initializedDefaultProvider; } /// - /// The CreateAssocation method creates an association between two objects. + /// The CreateAssociation method creates an association between two objects. /// Once an association is created, a designer or other filtering mechanism /// can add properties that route to either object into the primary object's /// property set. When a property invocation is made against the primary - /// object, GetAssocation will be called to resolve the actual object + /// object, GetAssociation will be called to resolve the actual object /// instance that is related to its type parameter. /// [EditorBrowsable(EditorBrowsableState.Advanced)] diff --git a/src/libraries/System.ComponentModel.TypeConverter/tests/TypeDescriptorTests.cs b/src/libraries/System.ComponentModel.TypeConverter/tests/TypeDescriptorTests.cs index 299d73cfe31..c8d85842d86 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/tests/TypeDescriptorTests.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/tests/TypeDescriptorTests.cs @@ -5,6 +5,10 @@ using System.Collections.Generic; using System.ComponentModel.Design; using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Moq; using Xunit; @@ -1231,5 +1235,248 @@ protected DerivedCultureInfo() : base("hello") class TwiceDerivedCultureInfo : DerivedCultureInfo { } + + private long _concurrentError = 0; + private bool ConcurrentError + { + get => Interlocked.Read(ref _concurrentError) == 1; + set => Interlocked.Exchange(ref _concurrentError, value ? 1 : 0); + } + + private void ConcurrentTest(TypeWithProperty instance) + { + var properties = TypeDescriptor.GetProperties(instance); + Thread.Sleep(10); + if (properties.Count > 0) + { + ConcurrentError = true; + } + } + + [SkipOnPlatform(TestPlatforms.Browser, "Thread.Start is not supported on browsers.")] + [Fact] + public void ConcurrentGetProperties_ReturnsExpected() + { + const int Timeout = 60000; + int concurrentCount = Environment.ProcessorCount * 2; + + using var finished = new CountdownEvent(concurrentCount); + + var instances = new TypeWithProperty[concurrentCount]; + for (int i = 0; i < concurrentCount; i++) + { + instances[i] = new TypeWithProperty(); + } + + for (int i = 0; i < concurrentCount; i++) + { + int i2 = i; + new Thread(() => + { + ConcurrentTest(instances[i2]); + finished.Signal(); + }).Start(); + } + + finished.Wait(Timeout); + + if (finished.CurrentCount != 0) + { + Assert.Fail("Timeout. Possible deadlock."); + } + else + { + Assert.False(ConcurrentError, "Fallback type descriptor is used. Possible race condition."); + } + } + + [SkipOnPlatform(TestPlatforms.Browser, "Thread.Start is not supported on browsers.")] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public static void ConcurrentAddProviderAndGetProvider() + { + // Use a timeout value lower than RemoteExecutor in order to get a nice Fail message. + const int Timeout = 50000; + + RemoteInvokeOptions options = new() + { + TimeOut = 60000 + }; + + RemoteExecutor.Invoke(() => + { + using var finished = new CountdownEvent(2); + Thread t1 = new Thread(() => + { + ConcurrentAddProvider(); + finished.Signal(); + }); + Thread t2 = new Thread(() => + { + ConcurrentGetProvider(); + finished.Signal(); + }); + t1.Start(); + t2.Start(); + finished.Wait(Timeout); + if (finished.CurrentCount != 0) + { + Assert.Fail("Timeout. Possible deadlock."); + } + }, options).Dispose(); + + static void ConcurrentAddProvider() + { + var provider = new EmptyPropertiesTypeProvider(); + TypeDescriptor.AddProvider(provider, typeof(MyClass)); + + // This test primarily verifies no deadlock, but verify the values anyway. + Assert.True(TypeDescriptor.GetProvider(typeof(MyClass)).IsSupportedType(typeof(MyClass))); + } + + static void ConcurrentGetProvider() + { + TypeDescriptionProvider provider = TypeDescriptor.GetProvider(typeof(TypeWithProperty)); + + // This test primarily verifies no deadlock, but verify the values anyway. + Assert.True(provider.IsSupportedType(typeof(TypeWithProperty))); + } + } + + public sealed class EmptyPropertiesTypeProvider : TypeDescriptionProvider + { + private sealed class EmptyPropertyListDescriptor : ICustomTypeDescriptor + { + public AttributeCollection GetAttributes() => AttributeCollection.Empty; + + public string? GetClassName() => null; + + public string? GetComponentName() => null; + + public TypeConverter? GetConverter() => new TypeConverter(); + + public EventDescriptor? GetDefaultEvent() => null; + + public PropertyDescriptor? GetDefaultProperty() => null; + + public object? GetEditor(Type editorBaseType) => null; + + public EventDescriptorCollection GetEvents() => EventDescriptorCollection.Empty; + + public EventDescriptorCollection GetEvents(Attribute[]? attributes) => GetEvents(); + + public PropertyDescriptorCollection GetProperties() => PropertyDescriptorCollection.Empty; + + public PropertyDescriptorCollection GetProperties(Attribute[]? attributes) => GetProperties(); + + public object? GetPropertyOwner(PropertyDescriptor? pd) => null; + } + public override ICustomTypeDescriptor GetTypeDescriptor(Type objectType, object? instance) + { + return new EmptyPropertyListDescriptor(); + } + } + + [TypeDescriptionProvider(typeof(EmptyPropertiesTypeProvider))] + public sealed class TypeWithProperty + { + public int OneProperty { get; set; } + } + + public static IEnumerable GetConverter_ByMultithread_ReturnsExpected_TestData() + { + yield return new object[] { typeof(MyClass), typeof(MyTypeConverter) }; + yield return new object[] { typeof(MyInheritedClassWithCustomTypeDescriptionProvider), typeof(MyInheritedClassWithCustomTypeDescriptionProviderConverter) }; + yield return new object[] { typeof(MyInheritedClassWithInheritedTypeDescriptionProvider), typeof(MyTypeConverter) }; + } + + [Theory] + [MemberData(nameof(GetConverter_ByMultithread_ReturnsExpected_TestData))] + public async void GetConverter_ByMultithread_ReturnsExpected(Type typeForGetConverter, Type expectedConverterType) + { + TypeConverter[] actualConverters = await Task.WhenAll( + Enumerable.Range(0, 100).Select(_ => + Task.Run(() => TypeDescriptor.GetConverter(typeForGetConverter)))); + Assert.All(actualConverters, + currentConverter => Assert.IsType(expectedConverterType, currentConverter)); + } + + public static IEnumerable GetConverterWithAddProvider_ByMultithread_Success_TestData() + { + foreach (object[] currentTestCase in GetConverter_ByMultithread_ReturnsExpected_TestData()) + { + yield return currentTestCase; + } + } + + [Theory] + [MemberData(nameof(GetConverterWithAddProvider_ByMultithread_Success_TestData))] + public async void GetConverterWithAddProvider_ByMultithread_Success(Type typeForGetConverter, Type expectedConverterType) + { + TypeConverter[] actualConverters = await Task.WhenAll( + Enumerable.Range(0, 200).Select(_ => + Task.Run(() => + { + var mockProvider = new Mock(MockBehavior.Strict); + var someInstance = new object(); + TypeDescriptor.AddProvider(mockProvider.Object, someInstance); + return TypeDescriptor.GetConverter(typeForGetConverter); + }))); + Assert.All(actualConverters, + currentConverter => Assert.IsType(expectedConverterType, currentConverter)); + } + + [TypeDescriptionProvider(typeof(MyClassTypeDescriptionProvider))] + public class MyClass + { + } + + [TypeDescriptionProvider(typeof(MyInheritedClassWithCustomTypeDescriptionProviderTypeDescriptionProvider))] + public class MyInheritedClassWithCustomTypeDescriptionProvider : MyClass + { + } + + public class MyInheritedClassWithInheritedTypeDescriptionProvider : MyClass + { + } + + public class MyClassTypeDescriptionProvider : TypeDescriptionProvider + { + public override ICustomTypeDescriptor GetTypeDescriptor(Type objectType, object instance) + { + return new MyClassTypeDescriptor(); + } + } + + public class MyInheritedClassWithCustomTypeDescriptionProviderTypeDescriptionProvider : TypeDescriptionProvider + { + public override ICustomTypeDescriptor GetTypeDescriptor(Type objectType, object instance) + { + return new MyInheritedClassWithCustomTypeDescriptionProviderTypeDescriptor(); + } + } + + public class MyClassTypeDescriptor : CustomTypeDescriptor + { + public override TypeConverter GetConverter() + { + return new MyTypeConverter(); + } + } + + public class MyInheritedClassWithCustomTypeDescriptionProviderTypeDescriptor : CustomTypeDescriptor + { + public override TypeConverter GetConverter() + { + return new MyInheritedClassWithCustomTypeDescriptionProviderConverter(); + } + } + + public class MyTypeConverter : TypeConverter + { + } + + public class MyInheritedClassWithCustomTypeDescriptionProviderConverter : TypeConverter + { + } } } From 11c5492e450eaa810bd78a2f9020215b29d63638 Mon Sep 17 00:00:00 2001 From: Rich Lander Date: Wed, 24 Apr 2024 08:54:23 -0700 Subject: [PATCH 086/151] Update EOL versions in release/8.0-staging (#101073) * Update EOL OS versions * Revert missing images * Revert missing images --- .../templates/pipeline-with-resources.yml | 6 ++-- .../coreclr/templates/helix-queues-setup.yml | 14 ++++---- .../libraries/helix-queues-setup.yml | 36 +++++++++---------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index 02242394fba..edba5767b61 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -55,9 +55,9 @@ resources: - container: linux_x64_dev_innerloop image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - # We use a CentOS Stream 8 image here to test building from source on CentOS Stream 8. + # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. - container: SourceBuild_centos_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. - container: SourceBuild_linux_x64 @@ -82,7 +82,7 @@ resources: image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 - container: linux_x64_llvmaot - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - container: browser_wasm image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 9a57f32f5e6..378396bc323 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -52,7 +52,7 @@ jobs: # Browser wasm - ${{ if eq(parameters.platform, 'browser_wasm') }}: - - (Ubuntu.1804.Amd64)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly + - (Ubuntu.1804.Amd64)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly # iOS devices - ${{ if in(parameters.platform, 'ios_arm64') }}: @@ -65,16 +65,16 @@ jobs: # Linux arm - ${{ if eq(parameters.platform, 'linux_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Ubuntu.1804.Arm32.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7 + - (Ubuntu.1804.Arm32.Open)Ubuntu.2204.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7 + - (Ubuntu.1804.Arm32)Ubuntu.2204.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7 # Linux arm64 - ${{ if eq(parameters.platform, 'linux_arm64') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Ubuntu.1804.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8 + - (Ubuntu.2204.Arm64.Open)Ubuntu.2204.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-arm64v8 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8 + - (Ubuntu.1804.Arm64)Ubuntu.2204.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8 # Linux musl x64 - ${{ if eq(parameters.platform, 'linux_musl_x64') }}: @@ -100,9 +100,9 @@ jobs: # Linux x64 - ${{ if eq(parameters.platform, 'linux_x64') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - Ubuntu.1804.Amd64.Open + - Ubuntu.2004.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - Ubuntu.1804.Amd64 + - Ubuntu.2004.Amd64 # OSX arm64 - ${{ if eq(parameters.platform, 'osx_arm64') }}: diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 8acabe6ec38..d3d7e7b2e64 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -28,21 +28,21 @@ jobs: # Linux arm - ${{ if eq(parameters.platform, 'linux_arm') }}: - ${{ if or(eq(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Debian.11.Arm32.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm32v7 + - (Debian.11.Arm32.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm32v7 # Linux armv6 - ${{ if eq(parameters.platform, 'linux_armv6') }}: # - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: - - (Raspbian.10.Armv6.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:raspbian-10-helix-arm32v6 + - (Raspbian.10.Armv6.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:raspbian-10-helix-arm32v6 # Linux arm64 - ${{ if eq(parameters.platform, 'linux_arm64') }}: - ${{ if or(eq(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Ubuntu.2204.Arm64.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-arm64v8 + - (Ubuntu.2204.Arm64.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-arm64v8 - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Ubuntu.1804.Arm64.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8 + - (Ubuntu.1804.Arm64.Open)Ubuntu.2204.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8 - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Debian.11.Arm64.Open)Ubuntu.1804.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8 + - (Debian.11.Arm64.Open)Ubuntu.2204.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-arm64v8 # Linux musl x64 - ${{ if eq(parameters.platform, 'linux_musl_x64') }}: @@ -61,25 +61,25 @@ jobs: - ${{ if and(eq(parameters.jobParameters.interpreter, ''), ne(parameters.jobParameters.isSingleFile, true)) }}: - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - SLES.15.Amd64.Open - - (Centos.8.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-helix - - (Fedora.38.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38-helix - - (Ubuntu.2204.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64 + - (Centos.9.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9-helix + - (Fedora.39.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39-helix + - (Ubuntu.2204.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-helix-amd64 - (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - ${{ if or(ne(parameters.jobParameters.testScope, 'outerloop'), ne(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - ${{ if or(eq(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - SLES.15.Amd64.Open - - (Fedora.38.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-38-helix + - (Fedora.39.Amd64.Open)Ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-39-helix - Ubuntu.2204.Amd64.Open - - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - - (Mariner.2.0.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-helix-amd64 - - (openSUSE.15.2.Amd64.Open)ubuntu.1804.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 + - (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 + - (Mariner.2.0.Amd64.Open)Ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-helix-amd64 + - (openSUSE.15.2.Amd64.Open)Ubuntu.2204.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:opensuse-15.2-helix-amd64 - ${{ if or(ne(parameters.jobParameters.isExtraPlatforms, true), eq(parameters.jobParameters.includeAllPlatforms, true)) }}: - - (Centos.8.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8-helix - - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 + - (Centos.9.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9-helix + - (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - Ubuntu.1804.Amd64.Open - ${{ if or(eq(parameters.jobParameters.interpreter, 'true'), eq(parameters.jobParameters.isSingleFile, true)) }}: # Limiting interp runs as we don't need as much coverage. - - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 + - (Debian.11.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 # Linux s390x - ${{ if eq(parameters.platform, 'linux_s390x') }}: @@ -171,15 +171,15 @@ jobs: # WASI - ${{ if eq(parameters.platform, 'wasi_wasm') }}: - - (Ubuntu.2004.Amd64)Ubuntu.2004.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-helix-wasm-amd64 + - (Ubuntu.2004.Amd64)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-helix-wasm-amd64 # Browser WebAssembly - ${{ if eq(parameters.platform, 'browser_wasm') }}: - - (Ubuntu.1804.Amd64)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly + - (Ubuntu.1804.Amd64)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly # Browser WebAssembly Firefox - ${{ if eq(parameters.platform, 'browser_wasm_firefox') }}: - - (Ubuntu.1804.Amd64)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly + - (Ubuntu.1804.Amd64)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-webassembly # Browser WebAssembly windows - ${{ if in(parameters.platform, 'browser_wasm_win', 'wasi_wasm_win') }}: From 845cd4f6526b529a323c3e91fda37ba2768f6e42 Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:36:59 -0700 Subject: [PATCH 087/151] Merge #91646 and #91708 into release/8.0-staging (#101313) * Move CoreCLR community platforms to build with the global build template (#91646) * Try to disable VSIX update service and collect VS update logs (#91708) * Tweak invocations, disable service updates, detect vswhere missing * Try disabling update early in build * Only collect logs on official build on exit --------- Co-authored-by: Jeremy Koritzinsky Co-authored-by: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> --- eng/build.sh | 2 +- eng/collect_vsinfo.ps1 | 65 +++++++++++++++++++ eng/disable_vsupdate.ps1 | 23 +++++++ eng/native/init-vs-env.cmd | 6 ++ eng/pipelines/common/global-build-job.yml | 5 ++ eng/pipelines/common/platform-matrix.yml | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 28 +++----- eng/pipelines/installer/jobs/build-job.yml | 6 ++ eng/pipelines/mono/templates/build-job.yml | 4 ++ eng/pipelines/runtime.yml | 44 ++++++++++++- 10 files changed, 163 insertions(+), 22 deletions(-) create mode 100644 eng/collect_vsinfo.ps1 create mode 100644 eng/disable_vsupdate.ps1 diff --git a/eng/build.sh b/eng/build.sh index 772d7ac8be8..4d0d17a2204 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -32,7 +32,7 @@ usage() echo " [Default: Debug]" echo " --os Target operating system: windows, linux, freebsd, osx, maccatalyst, tvos," echo " tvossimulator, ios, iossimulator, android, browser, wasi, netbsd, illumos, solaris" - echo " linux-musl, linux-bionic or haiku." + echo " linux-musl, linux-bionic, tizen, or haiku." echo " [Default: Your machine's OS.]" echo " --outputrid Optional argument that overrides the target rid name." echo " --projects Project or solution file(s) to build." diff --git a/eng/collect_vsinfo.ps1 b/eng/collect_vsinfo.ps1 new file mode 100644 index 00000000000..e2178fe4b24 --- /dev/null +++ b/eng/collect_vsinfo.ps1 @@ -0,0 +1,65 @@ +<# +.PARAMETER ArchiveRunName +Name of the run for vs logs + +.NOTES +Returns 0 if succeeds, 1 otherwise +#> +[CmdletBinding(PositionalBinding=$false)] +Param ( + [Parameter(Mandatory=$True)] + [string] $ArchiveRunName +) + +. $PSScriptRoot/common/tools.ps1 + +$ProgressPreference = "SilentlyContinue" +$LogDir = Join-Path $LogDir $ArchiveRunName +mkdir $LogDir + +$vscollect_uri="http://aka.ms/vscollect.exe" +$vscollect="$env:TEMP\vscollect.exe" + +if (-not (Test-Path $vscollect)) { + Retry({ + Write-Host "GET $vscollect_uri" + Invoke-WebRequest $vscollect_uri -OutFile $vscollect -UseBasicParsing + }) + + if (-not (Test-Path $vscollect)) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download vscollect." + exit 1 + } +} + +&"$vscollect" +Move-Item $env:TEMP\vslogs.zip "$LogDir" + +$vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path -Path "$vswhere" -PathType Leaf)) +{ + Write-Error "Couldn't locate vswhere at $vswhere" + exit 1 +} + +&"$vswhere" -all -prerelease -products * | Tee-Object -FilePath "$LogDir\vs_where.log" + +$vsdir = &"$vswhere" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + +if (-not (Test-Path $vsdir)) +{ + $procDumpDir = Join-Path $ToolsDir "procdump" + $procDumpToolPath = Join-Path $procDumpDir "procdump.exe" + $procdump_uri = "https://download.sysinternals.com/files/Procdump.zip" + + if (-not (Test-Path $procDumpToolPath)) { + Retry({ + Write-Host "GET $procdump_uri" + Invoke-WebRequest $procdump_uri -OutFile "$TempDir\Procdump.zip" -UseBasicParsing + }) + + Expand-Archive -Path "$TempDir\Procdump.zip" $procDumpDir + } + + &"$procDumpToolPath" -ma -accepteula VSIXAutoUpdate.exe "$LogDir" +} diff --git a/eng/disable_vsupdate.ps1 b/eng/disable_vsupdate.ps1 new file mode 100644 index 00000000000..d8d365f0577 --- /dev/null +++ b/eng/disable_vsupdate.ps1 @@ -0,0 +1,23 @@ +schtasks /change /tn "\Microsoft\VisualStudio\VSIX Auto Update" /disable + +$vswhere = "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" +if (-not (Test-Path -Path "$vswhere" -PathType Leaf)) +{ + Write-Error "Couldn't locate vswhere at $vswhere" + exit 1 +} + +$vsdir = &"$vswhere" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath +$vsregedit = "$vsdir\Common7\IDE\VsRegEdit.exe" + +if (-not (Test-Path -Path "$vsregedit" )) +{ + Write-Error "VSWhere returned path: $vsdir, but regedit $vsregedit doesn't exist." + exit 1 +} + +Write-Output "VSWhere returned path: $vsdir, using regedit $vsregedit" +Write-Output "Disabling updates through VS Registry:" + +&"$vsdir\Common7\IDE\VsRegEdit.exe" set local HKCU ExtensionManager AutomaticallyCheckForUpdates2Override dword 0 +&"$vsdir\Common7\IDE\VsRegEdit.exe" read local HKCU ExtensionManager AutomaticallyCheckForUpdates2Override dword diff --git a/eng/native/init-vs-env.cmd b/eng/native/init-vs-env.cmd index 273f49b3392..befe98ab407 100644 --- a/eng/native/init-vs-env.cmd +++ b/eng/native/init-vs-env.cmd @@ -26,6 +26,8 @@ if defined VisualStudioVersion goto :VSDetected set "__VSWhere=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" set "__VSCOMNTOOLS=" +if not exist "%__VSWhere%" goto :VSWhereMissing + if exist "%__VSWhere%" ( for /f "tokens=*" %%p in ( '"%__VSWhere%" -latest -prerelease -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath' @@ -56,6 +58,10 @@ echo %__MsgPrefix%Error: Visual Studio 2022 with C++ tools required. ^ Please see https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/windows-requirements.md for build requirements. exit /b 1 +:VSWhereMissing +echo %__MsgPrefix%Error: vswhere couldn not be found in Visual Studio Installer directory at "%__VSWhere%" +exit /b 1 + :SetVCEnvironment if "%__VCBuildArch%"=="" exit /b 0 diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 39e7d9b5c53..0d2ceac932d 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -254,6 +254,11 @@ jobs: shouldContinueOnError: ${{ parameters.shouldContinueOnError }} ${{ insert }}: ${{ parameters.extraStepsParameters }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 8399c82ecd7..c2f754d6ee2 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -883,7 +883,7 @@ jobs: jobTemplate: ${{ parameters.jobTemplate }} helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} variables: ${{ parameters.variables }} - osGroup: tizen + osGroup: linux # Our build scripts don't support Tizen and have always used Linux as the OS parameter. archType: armel targetRid: tizen-armel platform: tizen_armel diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 2e128ea54e1..01d12e6452b 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -95,9 +95,6 @@ jobs: - ${{ if ne(parameters.testGroup, 'innerloop') }}: - name: clrRuntimeComponentsBuildArg value: '-component runtime -component alljits -component paltests -component nativeaot -component spmi ' - - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}: - - name: clrRuntimeComponentsBuildArg - value: '-component runtime -component jit -component iltools -component spmi ' - name: pgoInstrumentArg value: '' @@ -108,12 +105,6 @@ jobs: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] - - name: clrRuntimePortableBuildArg - value: '' - - ${{ if eq(parameters.osGroup, 'tizen') }}: - - name: clrRuntimePortableBuildArg - value: '-portablebuild=false' - # Set a default empty argument for the pgo path. # This will be set during the 'native prerequisites' step if PGO optimization is enabled. - name: CoreClrPgoDataArg @@ -128,7 +119,6 @@ jobs: - ${{ parameters.variables }} steps: - # Install native dependencies # Linux builds use docker images with dependencies preinstalled, # and FreeBSD builds use a build agent with dependencies @@ -185,14 +175,14 @@ jobs: # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(clrRuntimePortableBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) $(clrRuntimePortableBuildArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -hostarch x64 $(osArg) -ci $(compilerArg) -component crosscomponents -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" $(officialBuildIdArg) displayName: Build CoreCLR Cross-Arch Tools (Tools that run on x64 targeting x86) - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: @@ -202,12 +192,8 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - ${{ if or(ne(parameters.osGroup, 'linux'), ne(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages - - ${{ if and(eq(parameters.osGroup, 'linux'), eq(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib $(crossArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci - displayName: Build managed product components and packages + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci + displayName: Build managed product components and packages # Build native test components - ${{ if and(ne(parameters.isOfficialBuild, true), ne(parameters.disableClrTest, true)) }}: @@ -295,6 +281,12 @@ jobs: parameters: name: ${{ parameters.platform }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + + # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 1d89cfad70e..3e18fb3d963 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -370,6 +370,12 @@ jobs: displayName: Build and Package continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() + + - ${{ if in(parameters.osGroup, 'osx', 'ios', 'tvos') }}: - script: | du -sh $(Build.SourcesDirectory)/* diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index 845078804fb..0c823e8d573 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -171,6 +171,10 @@ jobs: - script: build$(scriptExt) -subset mono+clr.hosts -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg) displayName: Build nupkg + - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: + - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log + displayName: Collect vslogs on exit + condition: always() # Publish Logs - task: PublishPipelineArtifact@1 displayName: Publish Logs diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 3aa0b650481..f2657982aee 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -70,7 +70,6 @@ extends: jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - - linux_x86 - linux_x64 - linux_arm - linux_arm64 @@ -79,7 +78,6 @@ extends: - linux_musl_arm64 - linux_musl_x64 - osx_arm64 - - tizen_armel - windows_x86 - windows_x64 - windows_arm64 @@ -188,6 +186,48 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_jit.containsChange'], true), eq(variables['isRollingBuild'], true))) + # + # Build CoreCLR with no R2R + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_x86 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NoR2R + buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR as a non-portable build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - tizen_armel + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NonPortable + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed From b86ccd0c240dc19aafff43387e461ac568c71276 Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Sun, 28 Apr 2024 15:05:42 -0700 Subject: [PATCH 088/151] Backport #89953, #91658 and #91643 to release/8.0-staging (#101579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Stop skipping tests in installer pipeline on linux_x64 (#89953) - Run tests in in installer pipeline on Linux_x64 even though it is cross-build. - Fix uploading of binaries/symbols on failure * Append job attempt number to log artifact names (#91658) Avoids errors like "Artifact Installer-Logs-coreclr--windows_arm64-Debug already exists for build 1636423." when retrying jobs. * Separate running PAL tests out to their own jobs/steps. (#91643) * Separate running PAL tests out to their own jobs/steps. * Remove PAL tests running infra from the CoreCLR build job and the runtime tests templates. * !drop Strip out all jobs from runtime.yml and replace with the new PAL tests job to make it easier to test * Remove dependency on EvaluatePaths * Reparent job parameters correctly. * Add back EvaluatePaths step for simplicity * Set missing properties * Capitalize build config * Make a tarball to preserve permission bits. * Make the output path directory before trying to use it * Double timeout * Double timeout again. Looks like Mac needs it. * Fix metadata name and put back to 10 minutes. * Move changes to the correct pipeline and revert changes to runtime.yml * Remove reference to deleted target --------- Co-authored-by: Elinor Fung Co-authored-by: Alexander Köplinger Co-authored-by: Jeremy Koritzinsky --- eng/pipelines/common/global-build-job.yml | 4 +- .../templates/runtimes/build-test-job.yml | 2 +- .../templates/runtimes/run-test-job.yml | 6 +-- .../templates/runtimes/send-to-helix-step.yml | 2 - eng/pipelines/coreclr/ci.yml | 20 ++++++++ .../coreclr/templates/build-jit-job.yml | 2 +- eng/pipelines/coreclr/templates/build-job.yml | 6 +-- .../coreclr/templates/crossdac-pack.yml | 2 +- .../coreclr/templates/run-paltests-step.yml | 23 +++++++++ eng/pipelines/installer/jobs/build-job.yml | 13 ++++- .../jobs/steps/upload-job-artifacts.yml | 2 +- eng/pipelines/mono/templates/build-job.yml | 2 +- .../mono/templates/generate-offsets.yml | 2 +- .../mono/templates/workloads-build.yml | 2 +- src/coreclr/scripts/paltests.proj | 44 ++++++++++++++++ .../RidAssetResolution.cs | 50 +++++++++++++------ src/tests/Common/helixpublishwitharcade.proj | 15 ------ 17 files changed, 147 insertions(+), 50 deletions(-) create mode 100644 eng/pipelines/coreclr/templates/run-paltests-step.yml create mode 100644 src/coreclr/scripts/paltests.proj diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 0d2ceac932d..6491bb53685 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -265,8 +265,8 @@ jobs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' PublishLocation: Container ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} continueOnError: true condition: always() diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 2b9871f183f..8def518f8c7 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -149,6 +149,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' + artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index a137b7c796e..759011b02e3 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -341,11 +341,7 @@ jobs: timeoutPerTestInMinutes: $(timeoutPerTestInMinutes) timeoutPerTestCollectionInMinutes: $(timeoutPerTestCollectionInMinutes) - runCrossGen2: ${{ eq(parameters.readyToRun, true) }} - ${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.runtimeFlavor, 'coreclr')) }}: - runPALTestsDir: '$(coreClrProductRootFolderPath)/paltests' - compositeBuildMode: ${{ parameters.compositeBuildMode }} runInUnloadableContext: ${{ parameters.runInUnloadableContext }} tieringTest: ${{ parameters.tieringTest }} @@ -603,7 +599,7 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_$(LogNamePrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' + artifactName: '${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_$(LogNamePrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.testGroup }}' continueOnError: true condition: always() diff --git a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml index 97d565b3a4e..4be1378ab3c 100644 --- a/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml +++ b/eng/pipelines/common/templates/runtimes/send-to-helix-step.yml @@ -17,7 +17,6 @@ parameters: timeoutPerTestCollectionInMinutes: '' timeoutPerTestInMinutes: '' runCrossGen2: '' - runPALTestsDir: '' compositeBuildMode: false helixProjectArguments: '' runInUnloadableContext: '' @@ -58,7 +57,6 @@ steps: _LongRunningGcTests: ${{ parameters.longRunningGcTests }} _GcSimulatorTests: ${{ parameters.gcSimulatorTests }} _Scenarios: ${{ join(',', parameters.scenarios) }} - _PALTestsDir: ${{ parameters.runPALTestsDir }} _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }} _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }} RuntimeFlavor: ${{ parameters.runtimeFlavor }} diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index adf052b014f..17806e94407 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -155,3 +155,23 @@ extends: readyToRun: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + + # + # PAL Tests + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Debug + platforms: + - linux_x64 + - linux_musl_x64 + - linux_arm64 + - linux_musl_arm64 + - osx_x64 + - osx_arm64 + jobParameters: + buildArgs: -s clr.paltests+clr.paltestlist + nameSuffix: PALTests + extraStepsTemplate: /eng/pipelines/coreclr/templates/run-paltests-step.yml diff --git a/eng/pipelines/coreclr/templates/build-jit-job.yml b/eng/pipelines/coreclr/templates/build-jit-job.yml index 5566f228b2f..2f78d87bf81 100644 --- a/eng/pipelines/coreclr/templates/build-jit-job.yml +++ b/eng/pipelines/coreclr/templates/build-jit-job.yml @@ -132,6 +132,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index 01d12e6452b..ed8945f6975 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -94,7 +94,7 @@ jobs: value: '' - ${{ if ne(parameters.testGroup, 'innerloop') }}: - name: clrRuntimeComponentsBuildArg - value: '-component runtime -component alljits -component paltests -component nativeaot -component spmi ' + value: '-component runtime -component alljits -component nativeaot -component spmi ' - name: pgoInstrumentArg value: '' @@ -192,7 +192,7 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+clr.paltestlist $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci displayName: Build managed product components and packages # Build native test components @@ -292,6 +292,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/pipelines/coreclr/templates/crossdac-pack.yml b/eng/pipelines/coreclr/templates/crossdac-pack.yml index 7489ceeedad..520eb65fb33 100644 --- a/eng/pipelines/coreclr/templates/crossdac-pack.yml +++ b/eng/pipelines/coreclr/templates/crossdac-pack.yml @@ -70,6 +70,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'CrossDacPackagingLogs' + artifactName: 'CrossDacPackagingLogs_Attempt$(System.JobAttempt)' continueOnError: true condition: always() diff --git a/eng/pipelines/coreclr/templates/run-paltests-step.yml b/eng/pipelines/coreclr/templates/run-paltests-step.yml new file mode 100644 index 00000000000..3b3881986f7 --- /dev/null +++ b/eng/pipelines/coreclr/templates/run-paltests-step.yml @@ -0,0 +1,23 @@ +parameters: + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + helixQueues: '' # required -- Helix queues + buildConfig: '' # required -- build configuration + archType: '' # required -- targeting CPU architecture + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + dependOnEvaluatePaths: false + +steps: + - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml + parameters: + displayName: 'Send job to Helix' + helixBuild: $(Build.BuildNumber) + helixSource: $(_HelixSource) + helixType: 'build/tests/' + helixQueues: ${{ parameters.helixQueues }} + creator: dotnet-bot + helixProjectArguments: '$(Build.SourcesDirectory)/src/coreclr/scripts/paltests.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + archType: ${{ parameters.archType }} + shouldContinueOnError: ${{ parameters.continueOnError }} diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 3e18fb3d963..f10fc47a506 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -86,18 +86,26 @@ jobs: - name: OfficialBuildArg value: '' + # Explicitly enable tests for linux even though it is a cross build using mariner + # They still work in this configuration and until they run on Helix, it is our only + # linux test coverage in the installer pipeline - name: SkipTests value: ${{ or( not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), - eq(parameters.crossBuild, true), + and( + eq(parameters.crossBuild, true), + not(and( + eq(parameters.osGroup, 'linux'), + eq(parameters.osSubgroup, '')) + )), eq(parameters.pgoType, 'PGO')) }} - name: BuildAction value: -test - - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), eq(parameters.crossBuild, true), eq(parameters.pgoType, 'PGO')), true) }}: + - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), and(eq(parameters.crossBuild, true), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '')))), eq(parameters.pgoType, 'PGO')), true) }}: - name: BuildAction value: '' @@ -405,6 +413,7 @@ jobs: runtimeVariant: ${{ parameters.runtimeVariant }} isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }} pgoType: ${{ parameters.pgoType }} + skipTests: ${{ eq(variables.SkipTests, true) }} - ${{ if ne(parameters.osGroup, 'windows') }}: - script: set -x && df -h diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index 6029ba00571..b3fd5328bc4 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -71,6 +71,6 @@ steps: displayName: Publish BuildLogs inputs: targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Installer-Logs-${{parameters.pgoType }}${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) + artifactName: Installer-Logs_Attempt$(System.JobAttempt)-${{parameters.pgoType }}${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) continueOnError: true condition: always() diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index 0c823e8d573..aa966218330 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -180,6 +180,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Mono_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index ddc67f0cc0b..c68adfe67a9 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -87,6 +87,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Mono_Offsets_$(osGroup)$(osSubGroup)' + artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' continueOnError: true condition: always() diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index b962dcb65d1..4a50ed665e2 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -96,7 +96,7 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'WorkloadLogs' + artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' continueOnError: true condition: always() diff --git a/src/coreclr/scripts/paltests.proj b/src/coreclr/scripts/paltests.proj new file mode 100644 index 00000000000..167208f4ecb --- /dev/null +++ b/src/coreclr/scripts/paltests.proj @@ -0,0 +1,44 @@ + + + true + $(_Creator) + $(_HelixAccessToken) + $(_HelixBuild) + $(_HelixSource) + $(_HelixTargetQueues) + $(_HelixType) + + + + + + + + + + + + + + + + + @(PalTestArchive) + ./runpaltestshelix.sh + 0:10 + + + + diff --git a/src/installer/tests/HostActivation.Tests/DependencyResolution/RidAssetResolution.cs b/src/installer/tests/HostActivation.Tests/DependencyResolution/RidAssetResolution.cs index cb924267ca2..afbf30a0ba4 100644 --- a/src/installer/tests/HostActivation.Tests/DependencyResolution/RidAssetResolution.cs +++ b/src/installer/tests/HostActivation.Tests/DependencyResolution/RidAssetResolution.cs @@ -29,7 +29,23 @@ public class TestSetup // Expected behaviour of the test based on above settings public bool ShouldUseRidGraph => UseRidGraph == true; - public bool ShouldUseFallbackRid => ShouldUseRidGraph && (Rid == UnknownRid || !HasRidGraph); + + public bool? ShouldUseFallbackRid + { + get + { + if (!ShouldUseRidGraph) + return false; + + if (Rid == UnknownRid || !HasRidGraph) + return true; + + // We use the product RID graph for testing (for cases with a RID graph). If the test is running + // on a platform that isn't in that RID graph, we may end up with the fallback even when the RID + // graph is used and RID is not unknown. Value of null indicates this state. + return null; + } + } public override string ToString() => $""" {nameof(Rid)}: {(Rid ?? "")} @@ -623,18 +639,20 @@ protected override void RunTest( UpdateAppConfigForTest(app, setup, copyOnUpdate: false); - dotnet.Exec(app.AppDll) + var result = dotnet.Exec(app.AppDll) .EnableTracingAndCaptureOutputs() .RuntimeId(setup.Rid) - .Execute() - .Should().Pass() + .Execute(); + result.Should().Pass() .And.HaveResolvedAssembly(expected.IncludedAssemblyPaths, app) .And.NotHaveResolvedAssembly(expected.ExcludedAssemblyPaths, app) .And.HaveResolvedNativeLibraryPath(expected.IncludedNativeLibraryPaths, app) .And.NotHaveResolvedNativeLibraryPath(expected.ExcludedNativeLibraryPaths, app) .And.HaveReadRidGraph(setup.ShouldUseRidGraph) - .And.HaveUsedFallbackRid(setup.ShouldUseFallbackRid) .And.HaveUsedFrameworkProbe(dotnet.GreatestVersionSharedFxPath, level: 1); + + if (setup.ShouldUseFallbackRid.HasValue) + result.Should().HaveUsedFallbackRid(setup.ShouldUseFallbackRid.Value); } } } @@ -674,17 +692,19 @@ protected override void RunTest( TestApp app = UpdateAppConfigForTest(SharedState.FrameworkReferenceApp, setup, copyOnUpdate: true); - SharedState.RunComponentResolutionTest(component.AppDll, app, dotnet.GreatestVersionHostFxrPath, command => command - .RuntimeId(setup.Rid)) - .Should().Pass() + var result = SharedState.RunComponentResolutionTest(component.AppDll, app, dotnet.GreatestVersionHostFxrPath, command => command + .RuntimeId(setup.Rid)); + result.Should().Pass() .And.HaveSuccessfullyResolvedComponentDependencies() .And.HaveResolvedComponentDependencyAssembly(expected.IncludedAssemblyPaths, component) .And.NotHaveResolvedComponentDependencyAssembly(expected.ExcludedAssemblyPaths, component) .And.HaveResolvedComponentDependencyNativeLibraryPath(expected.IncludedNativeLibraryPaths, component) .And.NotHaveResolvedComponentDependencyNativeLibraryPath(expected.ExcludedNativeLibraryPaths, component) .And.HaveReadRidGraph(setup.ShouldUseRidGraph) - .And.HaveUsedFallbackRid(setup.ShouldUseFallbackRid) .And.NotHaveUsedFrameworkProbe(dotnet.GreatestVersionSharedFxPath); + + if (setup.ShouldUseFallbackRid.HasValue) + result.Should().HaveUsedFallbackRid(setup.ShouldUseFallbackRid.Value); } } @@ -723,16 +743,18 @@ protected override void RunTest( app = UpdateAppConfigForTest(app, setup, copyOnUpdate: true); - SharedState.RunComponentResolutionTest(component.AppDll, app, app.Location, command => command - .RuntimeId(setup.Rid)) - .Should().Pass() + var result = SharedState.RunComponentResolutionTest(component.AppDll, app, app.Location, command => command + .RuntimeId(setup.Rid)); + result.Should().Pass() .And.HaveSuccessfullyResolvedComponentDependencies() .And.HaveResolvedComponentDependencyAssembly(expected.IncludedAssemblyPaths, component) .And.NotHaveResolvedComponentDependencyAssembly(expected.ExcludedAssemblyPaths, component) .And.HaveResolvedComponentDependencyNativeLibraryPath(expected.IncludedNativeLibraryPaths, component) .And.NotHaveResolvedComponentDependencyNativeLibraryPath(expected.ExcludedNativeLibraryPaths, component) - .And.HaveReadRidGraph(setup.ShouldUseRidGraph) - .And.HaveUsedFallbackRid(setup.ShouldUseFallbackRid); + .And.HaveReadRidGraph(setup.ShouldUseRidGraph); + + if (setup.ShouldUseFallbackRid.HasValue) + result.Should().HaveUsedFallbackRid(setup.ShouldUseFallbackRid.Value); } public class ComponentSharedTestState : ComponentSharedTestStateBase diff --git a/src/tests/Common/helixpublishwitharcade.proj b/src/tests/Common/helixpublishwitharcade.proj index db4ce658d6c..a439693adf2 100644 --- a/src/tests/Common/helixpublishwitharcade.proj +++ b/src/tests/Common/helixpublishwitharcade.proj @@ -46,7 +46,6 @@ <_RuntimeVariant> BundledNETCoreAppPackageVersion - <_PALTestsDir> <_SuperPmiCollect>false @@ -105,7 +104,6 @@ RuntimeVariant=$(_RuntimeVariant); BundledNETCoreAppPackageVersion=$(BundledNETCoreAppPackageVersion); HelixRuntimeRid=$(HelixRuntimeRid); - PALTestsDir=$(_PALTestsDir); SuperPmiCollect=$(_SuperPmiCollect) @@ -132,7 +130,6 @@ - <_Scenarios Include="$(_Scenarios.Split(','))" /> @@ -324,12 +321,6 @@ - - - - @@ -929,12 +920,6 @@ $(AppleTestTarget) $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - - - $(LegacyPayloadsRootDirectory)paltests.tar.gz - $(_WorkaroundForNuGetMigrationsForPrepending) ./runpaltestshelix.sh - $([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes))) - From 6eced1c4810a53e6c50e660298f31189bce2e3d0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 14:58:14 -0700 Subject: [PATCH 089/151] Update dependencies from https://github.com/dotnet/hotreload-utils build 20240429.2 (#101688) Microsoft.DotNet.HotReload.Utils.Generator.BuildTool From Version 8.0.0-alpha.0.24210.1 -> To Version 8.0.0-alpha.0.24229.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 04d355fd7b9..be70162daab 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -354,9 +354,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization 67613417f5e1af250e6ddfba79f8f2885d8e90fb - + https://github.com/dotnet/hotreload-utils - 85d6e21ac1d4e0977dfd1321131be0c912d70d80 + 61f137aacabdbd8f279415287a2dd70e150f5eb1 https://github.com/dotnet/runtime-assets diff --git a/eng/Versions.props b/eng/Versions.props index 3bf003d4987..34850e6ce95 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -186,7 +186,7 @@ 8.0.0-prerelease.24208.4 8.0.0-prerelease.24208.4 8.0.0-prerelease.24208.4 - 8.0.0-alpha.0.24210.1 + 8.0.0-alpha.0.24229.2 2.4.2 1.0.0 2.4.5 From 8ff4dd429a6d099ded84aebcf660649233e167fe Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 30 Apr 2024 10:57:23 +0200 Subject: [PATCH 090/151] Extend mono_gsharedvt_constrained_call for static calls and handle nullable value types (#101491) This PR extends mono_gsharedvt_constrained_call to handle static MONO_GSHAREDVT_CONSTRAINT_CALL_TYPE_REF calls. If the cmethod is a static method, this_arg should be NULL. Also, it skips dereferencing sharedvt ref arguments if they are nullable value types. --- src/mono/mono/mini/jit-icalls.c | 3 +- src/mono/mono/mini/method-to-ir.c | 7 +-- .../JitBlue/Runtime_94467/Runtime_94467.cs | 44 +++++++++++++++++++ .../Runtime_94467/Runtime_94467.csproj | 8 ++++ 4 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.csproj diff --git a/src/mono/mono/mini/jit-icalls.c b/src/mono/mono/mini/jit-icalls.c index 24ec1b65dcf..be43a91fdba 100644 --- a/src/mono/mono/mini/jit-icalls.c +++ b/src/mono/mono/mini/jit-icalls.c @@ -1450,7 +1450,8 @@ mono_gsharedvt_constrained_call (gpointer mp, MonoMethod *cmethod, MonoClass *kl break; case MONO_GSHAREDVT_CONSTRAINT_CALL_TYPE_REF: /* Calling a ref method with a ref receiver */ - this_arg = *(gpointer*)mp; + /* Static calls don't have this arg */ + this_arg = m_method_is_static (cmethod) ? NULL : *(gpointer*)mp; m = info->method; break; default: diff --git a/src/mono/mono/mini/method-to-ir.c b/src/mono/mono/mini/method-to-ir.c index eb0ab5f4e29..2dbb57653f3 100644 --- a/src/mono/mono/mini/method-to-ir.c +++ b/src/mono/mono/mini/method-to-ir.c @@ -3884,13 +3884,8 @@ handle_constrained_gsharedvt_call (MonoCompile *cfg, MonoMethod *cmethod, MonoMe int addr_reg; if (mini_is_gsharedvt_type (fsig->params [i])) { - MonoInst *is_deref; - int deref_arg_reg; ins = mini_emit_get_gsharedvt_info_klass (cfg, mono_class_from_mono_type_internal (fsig->params [i]), MONO_RGCTX_INFO_CLASS_BOX_TYPE); - deref_arg_reg = alloc_preg (cfg); - /* deref_arg = BOX_TYPE != MONO_GSHAREDVT_BOX_TYPE_VTYPE */ - EMIT_NEW_BIALU_IMM (cfg, is_deref, OP_ISUB_IMM, deref_arg_reg, ins->dreg, 1); - MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, is_gsharedvt_ins->dreg, i, is_deref->dreg); + MONO_EMIT_NEW_STORE_MEMBASE (cfg, OP_STOREI1_MEMBASE_REG, is_gsharedvt_ins->dreg, i, ins->dreg); } else if (has_gsharedvt) { MONO_EMIT_NEW_STORE_MEMBASE_IMM (cfg, OP_STOREI1_MEMBASE_IMM, is_gsharedvt_ins->dreg, i, 0); } diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.cs b/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.cs new file mode 100644 index 00000000000..85aa6e62655 --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.cs @@ -0,0 +1,44 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +public static class Runtime_94467 +{ + public interface ITypeChecker + { + static abstract bool Test(T value); + } + + public interface IHandler + { + bool Test(T value); + } + + public struct TypeChecker : ITypeChecker + { + public static bool Test(T value) => true; + } + + public class Handler : IHandler where TChecker : ITypeChecker + { + public bool Test(T value) => TChecker.Test(value); + } + + public static IHandler GetHandler() => new Handler(); + + [Fact] + public static int Test() + { + try { + var handler = GetHandler(); + if (handler.Test(true) && handler.Test(true)) + return 100; + else + return 101; + } catch (Exception) { + return -1; + } + } +} diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.csproj new file mode 100644 index 00000000000..15edd99711a --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_94467/Runtime_94467.csproj @@ -0,0 +1,8 @@ + + + True + + + + + \ No newline at end of file From 190798472c7741b4487243c837cf6d76e935ded1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:35:52 -0700 Subject: [PATCH 091/151] [release/8.0-staging] Revert "FileConfigurationProvider.Dispose should dispose FileProvider when it owns it" (#101610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revert "FileConfigurationProvider.Dispose should dispose FileProvider when it…" This reverts commit 63fad3c36fdf36e31ab0dd4c7e32732750390c4a. * Add test to ensure the bug does not come back * Enable servicing Microsoft.Extensions.Configuration.FileExtensions * Update Microsoft.Extensions.Configuration.FileExtensions.csproj --------- Co-authored-by: Adam Sitnik Co-authored-by: Eric StJohn --- .../src/FileConfigurationExtensions.cs | 10 +-- .../src/FileConfigurationProvider.cs | 5 -- .../src/FileConfigurationSource.cs | 11 ---- ...nsions.Configuration.FileExtensions.csproj | 2 + .../tests/JsonConfigurationTest.cs | 48 +++------------ .../tests/XmlConfigurationTest.cs | 61 ------------------- 6 files changed, 17 insertions(+), 120 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationExtensions.cs b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationExtensions.cs index 77ac387d096..f84c5c10eea 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationExtensions.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationExtensions.cs @@ -29,9 +29,6 @@ public static IConfigurationBuilder SetFileProvider(this IConfigurationBuilder b return builder; } - internal static IFileProvider? GetUserDefinedFileProvider(this IConfigurationBuilder builder) - => builder.Properties.TryGetValue(FileProviderKey, out object? provider) ? (IFileProvider)provider : null; - /// /// Gets the default to be used for file-based providers. /// @@ -41,7 +38,12 @@ public static IFileProvider GetFileProvider(this IConfigurationBuilder builder) { ThrowHelper.ThrowIfNull(builder); - return GetUserDefinedFileProvider(builder) ?? new PhysicalFileProvider(AppContext.BaseDirectory ?? string.Empty); + if (builder.Properties.TryGetValue(FileProviderKey, out object? provider)) + { + return (IFileProvider)provider; + } + + return new PhysicalFileProvider(AppContext.BaseDirectory ?? string.Empty); } /// diff --git a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationProvider.cs b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationProvider.cs index c0d8c9f3412..d226051b1ab 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationProvider.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationProvider.cs @@ -162,11 +162,6 @@ private void HandleException(ExceptionDispatchInfo info) protected virtual void Dispose(bool disposing) { _changeTokenRegistration?.Dispose(); - - if (Source.OwnsFileProvider) - { - (Source.FileProvider as IDisposable)?.Dispose(); - } } } } diff --git a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationSource.cs b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationSource.cs index 60555b2c672..d58c265f406 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationSource.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/FileConfigurationSource.cs @@ -18,11 +18,6 @@ public abstract class FileConfigurationSource : IConfigurationSource /// public IFileProvider? FileProvider { get; set; } - /// - /// Set to true when was not provided by user and can be safely disposed. - /// - internal bool OwnsFileProvider { get; private set; } - /// /// The path to the file. /// @@ -63,11 +58,6 @@ public abstract class FileConfigurationSource : IConfigurationSource /// The . public void EnsureDefaults(IConfigurationBuilder builder) { - if (FileProvider is null && builder.GetUserDefinedFileProvider() is null) - { - OwnsFileProvider = true; - } - FileProvider ??= builder.GetFileProvider(); OnLoadException ??= builder.GetFileLoadExceptionHandler(); } @@ -91,7 +81,6 @@ public void ResolveFileProvider() } if (Directory.Exists(directory)) { - OwnsFileProvider = true; FileProvider = new PhysicalFileProvider(directory); Path = pathToFile; } diff --git a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/Microsoft.Extensions.Configuration.FileExtensions.csproj b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/Microsoft.Extensions.Configuration.FileExtensions.csproj index 124b99b6338..1a60e989751 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/Microsoft.Extensions.Configuration.FileExtensions.csproj +++ b/src/libraries/Microsoft.Extensions.Configuration.FileExtensions/src/Microsoft.Extensions.Configuration.FileExtensions.csproj @@ -4,6 +4,8 @@ $(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.0;$(NetFrameworkMinimum) true true + true + 1 Provides a base class for file-based configuration providers used with Microsoft.Extensions.Configuration and extension methods for configuring them. diff --git a/src/libraries/Microsoft.Extensions.Configuration.Json/tests/JsonConfigurationTest.cs b/src/libraries/Microsoft.Extensions.Configuration.Json/tests/JsonConfigurationTest.cs index 4b08c918fb8..08d59e30ea6 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Json/tests/JsonConfigurationTest.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.Json/tests/JsonConfigurationTest.cs @@ -222,56 +222,26 @@ public void ThrowFormatExceptionWhenFileIsEmpty() Assert.Contains("Could not parse the JSON file.", exception.Message); } - [Theory] - [InlineData(true)] - [InlineData(false)] - public void AddJsonFile_FileProvider_Gets_Disposed_When_It_Was_Not_Created_By_The_User(bool disposeConfigRoot) + [Fact] + public void AddJsonFile_FileProvider_Is_Not_Disposed_When_SourcesGetReloaded() { - string filePath = Path.Combine(Path.GetTempPath(), $"{nameof(AddJsonFile_FileProvider_Gets_Disposed_When_It_Was_Not_Created_By_The_User)}.json"); + string filePath = Path.Combine(Path.GetTempPath(), $"{nameof(AddJsonFile_FileProvider_Is_Not_Disposed_When_SourcesGetReloaded)}.json"); File.WriteAllText(filePath, @"{ ""some"": ""value"" }"); - IConfigurationRoot config = new ConfigurationBuilder().AddJsonFile(filePath, optional: false).Build(); - JsonConfigurationProvider jsonConfigurationProvider = config.Providers.OfType().Single(); - - Assert.NotNull(jsonConfigurationProvider.Source.FileProvider); - PhysicalFileProvider fileProvider = (PhysicalFileProvider)jsonConfigurationProvider.Source.FileProvider; - Assert.False(GetIsDisposed(fileProvider)); - - if (disposeConfigRoot) - { - (config as IDisposable).Dispose(); // disposing ConfigurationRoot - } - else - { - jsonConfigurationProvider.Dispose(); // disposing JsonConfigurationProvider - } - - Assert.True(GetIsDisposed(fileProvider)); - } + IConfigurationBuilder builder = new ConfigurationManager(); - [Fact] - public void AddJsonFile_FileProvider_Is_Not_Disposed_When_It_Is_Owned_By_The_User() - { - string filePath = Path.Combine(Path.GetTempPath(), $"{nameof(AddJsonFile_FileProvider_Is_Not_Disposed_When_It_Is_Owned_By_The_User)}.json"); - File.WriteAllText(filePath, @"{ ""some"": ""value"" }"); + builder.AddJsonFile(filePath, optional: false); - PhysicalFileProvider fileProvider = new(Path.GetDirectoryName(filePath)); - JsonConfigurationProvider configurationProvider = new(new JsonConfigurationSource() - { - Path = filePath, - FileProvider = fileProvider - }); - IConfigurationRoot config = new ConfigurationBuilder().AddJsonFile(configurationProvider.Source.FileProvider, filePath, optional: true, reloadOnChange: false).Build(); + FileConfigurationSource fileConfigurationSource = (FileConfigurationSource)builder.Sources.Last(); + PhysicalFileProvider fileProvider = (PhysicalFileProvider)fileConfigurationSource.FileProvider; Assert.False(GetIsDisposed(fileProvider)); - (config as IDisposable).Dispose(); // disposing ConfigurationRoot that does not own the provider - Assert.False(GetIsDisposed(fileProvider)); + builder.Properties.Add("simplest", "repro"); - configurationProvider.Dispose(); // disposing JsonConfigurationProvider that does not own the provider Assert.False(GetIsDisposed(fileProvider)); - fileProvider.Dispose(); // disposing PhysicalFileProvider itself + fileProvider.Dispose(); Assert.True(GetIsDisposed(fileProvider)); } diff --git a/src/libraries/Microsoft.Extensions.Configuration.Xml/tests/XmlConfigurationTest.cs b/src/libraries/Microsoft.Extensions.Configuration.Xml/tests/XmlConfigurationTest.cs index d248d96d9d4..4012d775afa 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Xml/tests/XmlConfigurationTest.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.Xml/tests/XmlConfigurationTest.cs @@ -3,13 +3,11 @@ using System; using System.IO; -using System.Linq; using System.Security.Cryptography; using System.Security.Cryptography.Xml; using System.Tests; using System.Xml; using Microsoft.Extensions.Configuration.Test; -using Microsoft.Extensions.FileProviders; using Xunit; namespace Microsoft.Extensions.Configuration.Xml.Test @@ -781,64 +779,5 @@ public void LoadKeyValuePairsFromValidEncryptedXml() Assert.Equal("AnotherTestConnectionString", xmlConfigSrc.Get("data.setting:inventory:connectionstring")); Assert.Equal("MySql", xmlConfigSrc.Get("Data.setting:Inventory:Provider")); } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public void AddXmlFile_FileProvider_Gets_Disposed_When_It_Was_Not_Created_By_The_User(bool disposeConfigRoot) - { - string filePath = Path.Combine(Path.GetTempPath(), $"{nameof(AddXmlFile_FileProvider_Gets_Disposed_When_It_Was_Not_Created_By_The_User)}.xml"); - File.WriteAllText(filePath, @"Settings"); - - IConfigurationRoot config = new ConfigurationBuilder().AddXmlFile(filePath, optional: false).Build(); - XmlConfigurationProvider xmlConfigurationProvider = config.Providers.OfType().Single(); - - Assert.NotNull(xmlConfigurationProvider.Source.FileProvider); - PhysicalFileProvider fileProvider = (PhysicalFileProvider)xmlConfigurationProvider.Source.FileProvider; - Assert.False(GetIsDisposed(fileProvider)); - - if (disposeConfigRoot) - { - (config as IDisposable).Dispose(); // disposing ConfigurationRoot - } - else - { - xmlConfigurationProvider.Dispose(); // disposing XmlConfigurationProvider - } - - Assert.True(GetIsDisposed(fileProvider)); - } - - [Fact] - public void AddXmlFile_FileProvider_Is_Not_Disposed_When_It_Is_Owned_By_The_User() - { - string filePath = Path.Combine(Path.GetTempPath(), $"{nameof(AddXmlFile_FileProvider_Is_Not_Disposed_When_It_Is_Owned_By_The_User)}.xml"); - File.WriteAllText(filePath, @"Settings"); - - PhysicalFileProvider fileProvider = new(Path.GetDirectoryName(filePath)); - XmlConfigurationProvider configurationProvider = new(new XmlConfigurationSource() - { - Path = filePath, - FileProvider = fileProvider - }); - IConfigurationRoot config = new ConfigurationBuilder().AddXmlFile(configurationProvider.Source.FileProvider, filePath, optional: true, reloadOnChange: false).Build(); - - Assert.False(GetIsDisposed(fileProvider)); - - (config as IDisposable).Dispose(); // disposing ConfigurationRoot that does not own the provider - Assert.False(GetIsDisposed(fileProvider)); - - configurationProvider.Dispose(); // disposing XmlConfigurationProvider - Assert.False(GetIsDisposed(fileProvider)); - - fileProvider.Dispose(); // disposing PhysicalFileProvider itself - Assert.True(GetIsDisposed(fileProvider)); - } - - private static bool GetIsDisposed(PhysicalFileProvider fileProvider) - { - System.Reflection.FieldInfo isDisposedField = typeof(PhysicalFileProvider).GetField("_disposed", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); - return (bool)isDisposedField.GetValue(fileProvider); - } } } From c67a2e4216e156f4129638d71ea99a6a731106ed Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Tue, 30 Apr 2024 08:55:10 -0700 Subject: [PATCH 092/151] Enable repackaging of NETStandard 2.1 Targeting Pack (#101518) * Initial changes * Enable building in 8.0.6 release --- eng/Subsets.props | 1 + eng/Versions.props | 8 +++ src/installer/pkg/sfx/installers.proj | 4 ++ .../pkg/sfx/installers/netstandard2.1.proj | 60 +++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 src/installer/pkg/sfx/installers/netstandard2.1.proj diff --git a/eng/Subsets.props b/eng/Subsets.props index eb4151f3a18..bf0d6ca368c 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -507,6 +507,7 @@ + diff --git a/eng/Versions.props b/eng/Versions.props index d53eb6c7fd6..e071b668317 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -260,4 +260,12 @@ 8.0.101 $(MicrosoftDotnetSdkInternalVersion) + + + true + diff --git a/src/installer/pkg/sfx/installers.proj b/src/installer/pkg/sfx/installers.proj index 7f4ce6b9c1c..2f947e6d9e2 100644 --- a/src/installer/pkg/sfx/installers.proj +++ b/src/installer/pkg/sfx/installers.proj @@ -22,6 +22,10 @@ + + + + + + true + true + netstandard-targeting-pack-2.1 + NETStandard.Library.Ref + NETStandard.Library.Ref 2.1.0 + false + ToolPack + true + netstandard-targeting-pack-2.1.0-x64.rpm + https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/$(OriginalNETStandard21PkgFilename) + $([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'download')) + $(NETStandard21TempDir)$(OriginalNETStandard21PkgFilename) + $(NETStandard21TempDir)netstandard21.semaphore + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From ec346c0d36a8dffafffe7d6f541ec7afbfc6784b Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:58:03 -0700 Subject: [PATCH 093/151] Build the PGO build legs through the global build job template (#92296) (#101627) * Build the PGO build legs through the global build job template. * Remove pgoType parameters and plumbing from all jobs and use the -pgoinstrument flag instead of setting the property manually. * Limit subsets to build to limit packages and don't build crossgen2 sfxproj for internal optimization pipelines (we don't use it). * Fix job dependencies now that the non-global jobs don't have PGO type in the name * Change back to the triple-underscore name * Fix Mono artifacts to not have an extra underscore for PGO scenarios. * There's an extra underscore for the CoreCLR artifacts because there's no runtime variant. Co-authored-by: Jeremy Koritzinsky --- eng/Subsets.props | 2 +- eng/pipelines/common/global-build-job.yml | 3 -- .../templates/runtimes/build-test-job.yml | 1 - eng/pipelines/coreclr/templates/build-job.yml | 36 ++++++---------- .../superpmi-asmdiffs-checked-release-job.yml | 2 +- .../coreclr/templates/superpmi-diffs-job.yml | 2 +- .../coreclr/templates/xplat-pipeline-job.yml | 7 ++-- eng/pipelines/installer/jobs/build-job.yml | 29 ++++--------- .../jobs/steps/upload-job-artifacts.yml | 5 +-- eng/pipelines/libraries/base-job.yml | 3 +- eng/pipelines/mono/templates/build-job.yml | 1 - .../mono/templates/xplat-pipeline-job.yml | 8 ++-- eng/pipelines/runtime-official.yml | 41 ++++++------------- .../runtimelab-post-build-steps.yml | 3 +- 14 files changed, 48 insertions(+), 95 deletions(-) diff --git a/eng/Subsets.props b/eng/Subsets.props index bf0d6ca368c..00ae8a31c58 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -504,7 +504,7 @@ - + diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 6491bb53685..a13548956ff 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -14,7 +14,6 @@ parameters: dependsOn: [] pool: '' platform: '' - pgoType: '' condition: true useContinueOnErrorDuringBuild: false shouldContinueOnError: false @@ -134,7 +133,6 @@ jobs: targetRid: ${{ parameters.targetRid }} nameSuffix: ${{ parameters.nameSuffix }} platform: ${{ parameters.platform }} - pgoType: ${{ parameters.pgoType }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} ${{ if ne(variableTemplate.forwardedParameters, '') }}: ${{ each parameter in variableTemplate.forwardedParameters }}: @@ -250,7 +248,6 @@ jobs: targetRid: ${{ parameters.targetRid }} nameSuffix: ${{ parameters.nameSuffix }} platform: ${{ parameters.platform }} - pgoType: ${{ parameters.pgoType }} shouldContinueOnError: ${{ parameters.shouldContinueOnError }} ${{ insert }}: ${{ parameters.extraStepsParameters }} diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index 8def518f8c7..e249e8ac922 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -14,7 +14,6 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false crossBuild: false - pgoType: '' ### Build managed test components (native components are getting built as part ### of the product build job). diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index ed8945f6975..e995e626a56 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -17,7 +17,6 @@ parameters: testGroup: '' timeoutInMinutes: '' variables: {} - pgoType: '' ### Product build jobs: @@ -36,21 +35,18 @@ jobs: dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} disableComponentGovernance: ${{ parameters.disableComponentGovernance }} disableClrTest: ${{ parameters.disableClrTest }} - pgoType: ${{ parameters.pgoType }} # Compute job name from template parameters - name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}{5}', + name: ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, - parameters.buildConfig, - parameters.pgoType) }} - displayName: ${{ format('CoreCLR {0} Product Build {1}{2} {3} {4} {5}', + parameters.buildConfig) }} + displayName: ${{ format('CoreCLR {0} Product Build {1}{2} {3} {4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, - parameters.buildConfig, - parameters.pgoType) }} + parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -80,7 +76,7 @@ jobs: - name: enforcePgoArg value: '' # The EnforcePGO script is only supported on Windows and is not supported on arm64. - - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'arm64')), ne(parameters.pgoType, 'pgo')) }}: + - ${{ if and(eq(parameters.buildConfig, 'Release'), and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'arm64'))) }}: - name: enforcePgoArg value: '-enforcepgo' @@ -96,12 +92,6 @@ jobs: - name: clrRuntimeComponentsBuildArg value: '-component runtime -component alljits -component nativeaot -component spmi ' - - name: pgoInstrumentArg - value: '' - - ${{ if eq(parameters.pgoType, 'PGO' )}}: - - name: pgoInstrumentArg - value: '-pgoinstrument ' - - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] @@ -175,10 +165,10 @@ jobs: # Build CoreCLR Runtime - ${{ if ne(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) $(crossArg) $(osArg) -ci $(compilerArg) $(clrRuntimeComponentsBuildArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) $(nativeSymbols) displayName: Build CoreCLR Runtime - ${{ if eq(parameters.osGroup, 'windows') }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(pgoInstrumentArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) + - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) $(archType) -ci $(enforcePgoArg) $(officialBuildIdArg) $(clrInterpreterBuildArg) $(CoreClrPgoDataArg) displayName: Build CoreCLR Runtime - ${{ if or(eq(parameters.crossBuild, 'true'), ne(parameters.archType, 'x64')) }}: @@ -192,7 +182,7 @@ jobs: displayName: Disk Usage after Build # Build CoreCLR Managed Components - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(pgoInstrumentArg) $(officialBuildIdArg) -ci + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages $(crossArg) $(compilerArg) -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) -ci displayName: Build managed product components and packages # Build native test components @@ -241,7 +231,7 @@ jobs: artifactName: $(buildProductArtifactName) displayName: 'product build' - - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86'), eq(parameters.pgoType, '')) }}: + - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86')) }}: - template: /eng/pipelines/coreclr/templates/crossdac-build.yml parameters: archType: ${{ parameters.archType }} @@ -253,7 +243,7 @@ jobs: ${{ else }}: hostArchType: x64 - - ${{ if and(in(parameters.osGroup, 'windows'), eq(parameters.archType, 'x86'), eq(parameters.pgoType, '')) }}: + - ${{ if and(in(parameters.osGroup, 'windows'), eq(parameters.archType, 'x86')) }}: - template: /eng/pipelines/coreclr/templates/crossdac-build.yml parameters: archType: arm @@ -264,7 +254,7 @@ jobs: - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: # Publish test native components for consumption by test execution. - - ${{ if and(ne(parameters.isOfficialBuild, true), eq(parameters.pgoType, '')) }}: + - ${{ if ne(parameters.isOfficialBuild, true) }}: - template: /eng/pipelines/common/upload-artifact-step.yml parameters: rootFolder: $(nativeTestArtifactRootFolderPath) @@ -276,7 +266,7 @@ jobs: displayName: 'native test components' # Save packages using the prepare-signed-artifacts format. - - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.pgoType, '')) }}: + - ${{ if eq(parameters.isOfficialBuild, true) }}: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: name: ${{ parameters.platform }} @@ -292,6 +282,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + artifactName: '$(publishLogsArtifactPrefix)_Attempt$(System.JobAttempt)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' continueOnError: true condition: always() diff --git a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml index 396fe5077f5..659483c9bc3 100644 --- a/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-asmdiffs-checked-release-job.yml @@ -34,7 +34,7 @@ jobs: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: diff --git a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml index aace201eb96..9c2290c43a3 100644 --- a/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-diffs-job.yml @@ -49,7 +49,7 @@ jobs: - name: releaseProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release' - name: releaseProductArtifactName - value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release' steps: diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 83347d78b8b..162f5e9f0ed 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -38,7 +38,6 @@ jobs: crossBuild: ${{ parameters.crossBuild }} strategy: ${{ parameters.strategy }} pool: ${{ parameters.pool }} - pgoType: ${{ parameters.pgoType }} # arcade-specific parameters condition: and(succeeded(), ${{ parameters.condition }}) @@ -65,7 +64,7 @@ jobs: # Build product defines what we are trying to build, either coreclr or mono - name: buildProductArtifactName - value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' @@ -78,13 +77,13 @@ jobs: # We need this because both mono and coreclr build currently depends on CoreClr - name: coreClrProductArtifactName - value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: coreClrProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - name: corelibProductArtifactName - value: 'CoreLib_${{ parameters.pgoType }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'CoreLib_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: managedGenericTestArtifactName value: 'CoreCLRManagedTestArtifacts_AnyOS_AnyCPU_$(buildConfig)' diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index f10fc47a506..1c45b437c80 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -19,7 +19,6 @@ parameters: displayName: '' runtimeVariant: '' pool: '' - pgoType: '' runOnlyIfDependenciesSucceeded: false # The target names here should match container names in the resources section in our pipelines, like runtime.yml @@ -59,11 +58,10 @@ jobs: dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} disableClrTest: ${{ parameters.disableClrTest }} - pgoType: ${{ parameters.pgoType }} # Compute job name from template parameters - name: ${{ format('installer_{0}_{1}_{2}_{3}_{4}_', parameters.pgoType, parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} - displayName: ${{ format('{0} Installer Build and Test {1} {2} {3} {4}', parameters.pgoType, parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + name: ${{ format('installer_{0}_{1}_{2}_{3}_', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} + displayName: ${{ format('Installer Build and Test {0} {1} {2} {3}', parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }} # Run all steps in the container. # Note that the containers are defined in platform-matrix.yml @@ -99,25 +97,18 @@ jobs: not(and( eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '')) - )), - eq(parameters.pgoType, 'PGO')) }} + ))) }} - name: BuildAction value: -test - - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), and(eq(parameters.crossBuild, true), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, '')))), eq(parameters.pgoType, 'PGO')), true) }}: + - ${{ if eq(or(not(in(parameters.archType, 'x64', 'x86')), eq(parameters.runtimeFlavor, 'mono'), eq(parameters.isOfficialBuild, true), and(eq(parameters.crossBuild, true), not(and(eq(parameters.osGroup, 'linux'), eq(parameters.osSubgroup, ''))))), true) }}: - name: BuildAction value: '' - name: SignType value: test - - name: pgoInstrumentArg - value: '' - - ${{ if eq(parameters.pgoType, 'PGO' )}}: - - name: pgoInstrumentArg - value: '-pgoinstrument ' - # Set up non-PR build from internal project - ${{ if eq(parameters.isOfficialBuild, true) }}: - name: SignType @@ -153,7 +144,6 @@ jobs: build.cmd -subset host+packs -ci $(BuildAction) -configuration $(_BuildConfig) - $(pgoInstrumentArg) $(LiveOverridePathArgs) $(CommonMSBuildArgs) $(MsbuildSigningArguments) @@ -225,7 +215,6 @@ jobs: /p:CrossBuild=${{ parameters.crossBuild }} /p:PortableBuild=$(_PortableBuild) /p:SkipTests=$(SkipTests) - $(pgoInstrumentArg) $(LiveOverridePathArgs) $(CommonMSBuildArgs) @@ -265,7 +254,7 @@ jobs: /p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath) /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }} - name: RuntimeArtifactName - value: $(runtimeFlavorName)Product_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName) + value: $(runtimeFlavorName)Product_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName) - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - name: liveLibrariesLegName @@ -286,14 +275,13 @@ jobs: - evaluate_paths - ${{ parameters.dependsOn }} - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}: - - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}{6}', + - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}', parameters.runtimeFlavor, parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, - parameters.liveRuntimeBuildConfig, - parameters.pgoType) }} + parameters.liveRuntimeBuildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - libraries_build_${{ format('{0}{1}_{2}_{3}', parameters.osGroup, @@ -391,7 +379,7 @@ jobs: displayName: Disk Usage after Build # Only in glibc leg, we produce RPMs and Debs - - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'linux_x64'), eq(parameters.platform, 'linux_arm64')), eq(parameters.osSubgroup, ''), eq(parameters.pgoType, ''))}}: + - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'linux_x64'), eq(parameters.platform, 'linux_arm64')), eq(parameters.osSubgroup, ''))}}: - ${{ each packageBuild in parameters.packageDistroList }}: # This leg's RID matches the build image. Build its distro-dependent packages, as well as # the distro-independent installers. (There's no particular reason to build the distro- @@ -412,7 +400,6 @@ jobs: runtimeFlavor: ${{ parameters.runtimeFlavor }} runtimeVariant: ${{ parameters.runtimeVariant }} isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }} - pgoType: ${{ parameters.pgoType }} skipTests: ${{ eq(variables.SkipTests, true) }} - ${{ if ne(parameters.osGroup, 'windows') }}: diff --git a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml index b3fd5328bc4..4012b9a4fa3 100644 --- a/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml +++ b/eng/pipelines/installer/jobs/steps/upload-job-artifacts.yml @@ -3,7 +3,6 @@ parameters: runtimeFlavor: 'coreclr' runtimeVariant: '' isOfficialBuild: false - pgoType: '' steps: # Upload build artifacts (packages) to pipeline only if official, to save storage space. @@ -51,7 +50,7 @@ steps: displayName: Publish binaries inputs: pathtoPublish: '$(Build.StagingDirectory)/corehost-bin-${{ parameters.name }}-$(_BuildConfig)$(archiveExtension)' - artifactName: Installer-Binaries-${{parameters.pgoType }}${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) + artifactName: Installer-Binaries-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) continueOnError: true condition: failed() @@ -71,6 +70,6 @@ steps: displayName: Publish BuildLogs inputs: targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: Installer-Logs_Attempt$(System.JobAttempt)-${{parameters.pgoType }}${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) + artifactName: Installer-Logs_Attempt$(System.JobAttempt)-${{ parameters.runtimeFlavor }}-${{ parameters.runtimeVariant }}-${{ parameters.name }}-$(_BuildConfig) continueOnError: true condition: always() diff --git a/eng/pipelines/libraries/base-job.yml b/eng/pipelines/libraries/base-job.yml index 2448124a7bc..54fe98e7d85 100644 --- a/eng/pipelines/libraries/base-job.yml +++ b/eng/pipelines/libraries/base-job.yml @@ -23,7 +23,6 @@ parameters: testScope: '' pool: '' runTests: false - pgoType: '' jobs: - template: /eng/common/templates/job/job.yml @@ -98,7 +97,7 @@ jobs: - _runtimeDownloadPath: '$(Build.SourcesDirectory)/artifacts/transport/${{ parameters.runtimeFlavor }}' - _runtimeConfigurationArg: -rc ${{ parameters.liveRuntimeBuildConfig }} - ${{ if eq(parameters.runTests, true) }}: - - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.pgoType }}_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' + - _runtimeArtifactName: '$(runtimeFlavorName)Product_${{ parameters.runtimeVariant}}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.liveRuntimeBuildConfig }}' - _runtimeArtifactsPathArg: ' /p:RuntimeArtifactsPath=$(_runtimeDownloadPath)' - ${{ if eq(parameters.testDisplayName, '') }}: - _testRunNamePrefixSuffix: $(runtimeFlavorName)_${{ parameters.liveRuntimeBuildConfig }} diff --git a/eng/pipelines/mono/templates/build-job.yml b/eng/pipelines/mono/templates/build-job.yml index aa966218330..86e0813c7c7 100644 --- a/eng/pipelines/mono/templates/build-job.yml +++ b/eng/pipelines/mono/templates/build-job.yml @@ -15,7 +15,6 @@ parameters: dependsOn: [] monoCrossAOTTargetOS: [] dependOnEvaluatePaths: false - pgoType: '' ### Product build jobs: diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 2c369f71f30..1ca84d9caac 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -52,22 +52,22 @@ jobs: variables: - name: coreClrProductArtifactName - value: 'CoreCLRProduct___$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' + value: 'CoreCLRProduct__$(osGroup)$(osSubgroup)_$(archType)_${{ parameters.liveRuntimeBuildConfig }}' - name: coreClrProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(liveRuntimeBuildConfigUpper)' - name: buildProductArtifactName - value: 'MonoProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value: 'MonoProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' # minijit and monointerpreter do not use separate product builds. - ${{ if or(eq(parameters.runtimeVariant, 'minijit'), eq(parameters.runtimeVariant, 'monointerpreter')) }}: - name : buildProductArtifactName - value : 'MonoProduct___$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value : 'MonoProduct__$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - ${{ if eq(parameters.runtimeVariant, 'llvmfullaot') }}: - name : buildProductArtifactName - value : 'MonoProduct__llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + value : 'MonoProduct_llvmaot_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' - name: binTestsPath value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 3a9fd8d89ac..34658018fca 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -425,12 +425,13 @@ extends: - windows_arm64 # - # Build PGO CoreCLR release + # Build PGO Instrumented CoreCLR Release # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml - buildConfig: release + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: Release + helixQueueGroup: ci platforms: - windows_x64 - windows_x86 @@ -438,29 +439,13 @@ extends: - windows_arm64 - linux_arm64 jobParameters: + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument isOfficialBuild: ${{ variables.isOfficialBuild }} - signBinaries: false - testGroup: innerloop - pgoType: 'PGO' - - # - # PGO Build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - buildConfig: Release - jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - pgoType: 'PGO' - platforms: - - windows_x64 - - windows_x86 - - linux_x64 - - windows_arm64 - - linux_arm64 + nameSuffix: PGO + extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + extraStepsParameters: + name: PGO + timeoutInMinutes: 95 # # Build Workloads @@ -490,9 +475,9 @@ extends: - Build_tvossimulator_arm64_release_AllSubsets_Mono - Build_tvossimulator_x64_release_AllSubsets_Mono - Build_windows_x64_release_CrossAOT_Mono - - installer__coreclr__windows_x64_Release_ - - installer__coreclr__windows_x86_Release_ - - installer__coreclr__windows_arm64_Release_ + - installer_coreclr__windows_x64_Release_ + - installer_coreclr__windows_x86_Release_ + - installer_coreclr__windows_arm64_Release_ - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml diff --git a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml index a6187638cd3..97c007ba4b3 100644 --- a/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml +++ b/eng/pipelines/runtimelab/runtimelab-post-build-steps.yml @@ -5,7 +5,6 @@ parameters: osSubgroup: '' nameSuffix: '' platform: '' - pgoType: '' runtimeVariant: '' librariesBinArtifactName: '' isOfficialBuild: false @@ -32,7 +31,7 @@ steps: tarCompression: $(tarCompression) includeRootFolder: false archiveExtension: $(archiveExtension) - artifactName: CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} + artifactName: CoreCLRProduct_${{ parameters.runtimeVariant }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }} displayName: 'CoreCLR product build' # Zip Test Build From 8acc1b574817f9d9c6cd79ac5ab9a6348c4004d7 Mon Sep 17 00:00:00 2001 From: Jeff Handley Date: Tue, 30 Apr 2024 11:55:14 -0700 Subject: [PATCH 094/151] [release/8.0-staging] Fix #91958: use mach_timebase_info to determine process time coefficient on macOS (#100122) * Fix #91958: use mach_timebase_info to determine process time coefficient on macOS * (#91958) Process.OSX: fallback for non-zero exit code from mach_timebase_info * (#91958) Process: move OSX-specific tests to a separate file * (#91958) Process.OSX: calculate times lazily * (#91958) Process: include string resources properly in tests * (#91958) Process: more overflow-safe logic * (#91958) Minimize diff * Process.OSX: fix thread safety (#100260) Without this fix, it was possible for another thread to see an incorrect (zero) value of s_timeBase_numer because of a race condition. --------- Co-authored-by: Friedrich von Never --- .../src/Interop/OSX/Interop.Libraries.cs | 1 + .../src/Interop/OSX/Interop.libSystem.cs | 22 +++++++++++ .../src/System.Diagnostics.Process.csproj | 2 + .../src/System/Diagnostics/Process.OSX.cs | 37 +++++++++++++++++-- .../tests/ProcessTests.Unix.cs | 33 +++++++++++++++++ .../tests/Resources/Strings.resx | 6 +++ .../System.Diagnostics.Process.Tests.csproj | 6 +++ 7 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 src/libraries/Common/src/Interop/OSX/Interop.libSystem.cs diff --git a/src/libraries/Common/src/Interop/OSX/Interop.Libraries.cs b/src/libraries/Common/src/Interop/OSX/Interop.Libraries.cs index 36cdea9e1bd..40423ae3d20 100644 --- a/src/libraries/Common/src/Interop/OSX/Interop.Libraries.cs +++ b/src/libraries/Common/src/Interop/OSX/Interop.Libraries.cs @@ -11,6 +11,7 @@ internal static partial class Libraries internal const string libobjc = "/usr/lib/libobjc.dylib"; internal const string libproc = "/usr/lib/libproc.dylib"; internal const string Odbc32 = "libodbc.2.dylib"; + internal const string libSystem = "libSystem.dylib"; internal const string OpenLdap = "libldap.dylib"; internal const string SystemConfigurationLibrary = "/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration"; internal const string AppleCryptoNative = "libSystem.Security.Cryptography.Native.Apple"; diff --git a/src/libraries/Common/src/Interop/OSX/Interop.libSystem.cs b/src/libraries/Common/src/Interop/OSX/Interop.libSystem.cs new file mode 100644 index 00000000000..e8662da28e7 --- /dev/null +++ b/src/libraries/Common/src/Interop/OSX/Interop.libSystem.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class libSystem + { + [LibraryImport(Interop.Libraries.libSystem)] + public static unsafe partial int mach_timebase_info(mach_timebase_info_data_t* info); + public struct mach_timebase_info_data_t + { + public uint numer; + public uint denom; + } + } +} diff --git a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj index e82e069ffd3..ddd518ff333 100644 --- a/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj +++ b/src/libraries/System.Diagnostics.Process/src/System.Diagnostics.Process.csproj @@ -317,6 +317,8 @@ Link="Common\Interop\OSX\Interop.libproc.cs" /> + diff --git a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.OSX.cs b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.OSX.cs index da44fc1ed34..07f55780d82 100644 --- a/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.OSX.cs +++ b/src/libraries/System.Diagnostics.Process/src/System/Diagnostics/Process.OSX.cs @@ -10,6 +10,7 @@ namespace System.Diagnostics public partial class Process { private const int NanosecondsTo100NanosecondsFactor = 100; + private static volatile uint s_timeBase_numer, s_timeBase_denom; private const int MicrosecondsToSecondsFactor = 1_000_000; @@ -23,7 +24,7 @@ public TimeSpan PrivilegedProcessorTime { EnsureState(State.HaveNonExitedId); Interop.libproc.rusage_info_v3 info = Interop.libproc.proc_pid_rusage(_processId); - return new TimeSpan(Convert.ToInt64(info.ri_system_time / NanosecondsTo100NanosecondsFactor)); + return MapTime(info.ri_system_time); } } @@ -65,7 +66,7 @@ public TimeSpan TotalProcessorTime { EnsureState(State.HaveNonExitedId); Interop.libproc.rusage_info_v3 info = Interop.libproc.proc_pid_rusage(_processId); - return new TimeSpan(Convert.ToInt64((info.ri_system_time + info.ri_user_time) / NanosecondsTo100NanosecondsFactor)); + return MapTime(info.ri_system_time + info.ri_user_time); } } @@ -82,7 +83,7 @@ public TimeSpan UserProcessorTime { EnsureState(State.HaveNonExitedId); Interop.libproc.rusage_info_v3 info = Interop.libproc.proc_pid_rusage(_processId); - return new TimeSpan(Convert.ToInt64(info.ri_user_time / NanosecondsTo100NanosecondsFactor)); + return MapTime(info.ri_user_time); } } @@ -109,5 +110,35 @@ private static Interop.libproc.rusage_info_v3 GetCurrentProcessRUsage() { return Interop.libproc.proc_pid_rusage(Environment.ProcessId); } + + private static TimeSpan MapTime(ulong sysTime) + { + uint denom = s_timeBase_denom; + if (denom == default) + { + Interop.libSystem.mach_timebase_info_data_t timeBase = GetTimeBase(); + s_timeBase_numer = timeBase.numer; + s_timeBase_denom = denom = timeBase.denom; + } + uint numer = s_timeBase_numer; + + // By dividing by NanosecondsTo100NanosecondsFactor first, we lose some precision, but increase the range + // where no overflow will happen. + return new TimeSpan(Convert.ToInt64(sysTime / NanosecondsTo100NanosecondsFactor * numer / denom)); + } + + private static unsafe Interop.libSystem.mach_timebase_info_data_t GetTimeBase() + { + Interop.libSystem.mach_timebase_info_data_t timeBase = default; + var returnCode = Interop.libSystem.mach_timebase_info(&timeBase); + Debug.Assert(returnCode == 0, $"Non-zero exit code from mach_timebase_info: {returnCode}"); + if (returnCode != 0) + { + // Fallback: let's assume that the time values are in nanoseconds, + // i.e. the time base is 1/1. + timeBase.numer = timeBase.denom = 1; + } + return timeBase; + } } } diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.Unix.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.Unix.cs index 2085fddbb5d..79ad7ec01f0 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.Unix.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.Unix.cs @@ -851,6 +851,39 @@ static void ExecuteChildProcess(string filename, string flags) } } + [Fact] + [PlatformSpecific(TestPlatforms.OSX)] + public unsafe void TestTotalProcessorTimeMacOs() + { + var rUsage = Interop.libproc.proc_pid_rusage(Environment.ProcessId); + var timeBase = new Interop.libSystem.mach_timebase_info_data_t(); + Interop.libSystem.mach_timebase_info(&timeBase); + + var nativeUserUs = rUsage.ri_user_time / 1000 * timeBase.numer / timeBase.denom; + var nativeSystemUs = rUsage.ri_system_time / 1000 * timeBase.numer / timeBase.denom; + var nativeTotalUs = nativeSystemUs + nativeUserUs; + + var nativeUserTime = TimeSpan.FromMicroseconds(nativeUserUs); + var nativeSystemTime = TimeSpan.FromMicroseconds(nativeSystemUs); + var nativeTotalTime = TimeSpan.FromMicroseconds(nativeTotalUs); + + var process = Process.GetCurrentProcess(); + var managedUserTime = process.UserProcessorTime; + var managedSystemTime = process.PrivilegedProcessorTime; + var managedTotalTime = process.TotalProcessorTime; + + AssertTime(managedUserTime, nativeUserTime, "user"); + AssertTime(managedSystemTime, nativeSystemTime, "system"); + AssertTime(managedTotalTime, nativeTotalTime, "total"); + + void AssertTime(TimeSpan managed, TimeSpan native, string label) + { + Assert.True( + managed >= native, + $"Time '{label}' returned by managed API ({managed}) should be greated or equal to the time returned by native API ({native})."); + } + } + [ConditionalTheory(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] [InlineData(true)] [InlineData(false)] diff --git a/src/libraries/System.Diagnostics.Process/tests/Resources/Strings.resx b/src/libraries/System.Diagnostics.Process/tests/Resources/Strings.resx index 4fd9192b919..d443842f56c 100644 --- a/src/libraries/System.Diagnostics.Process/tests/Resources/Strings.resx +++ b/src/libraries/System.Diagnostics.Process/tests/Resources/Strings.resx @@ -120,4 +120,10 @@ The argv[0] argument cannot include a double quote. + + Could not get all running Process IDs. + + + Failed to set or retrieve rusage information. See the error code for OS-specific error information. + \ No newline at end of file diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index c3f01adc3bc..889e3434404 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -63,6 +63,12 @@ + + + From 47f25fbcd4a1e7b29d8ec7f672bdade9a34d7cfd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 30 Apr 2024 16:52:40 -0700 Subject: [PATCH 095/151] [release/8.0-staging] Prefer most derived member in Configuration Binder source generator (#101686) * Prefer most derived member in Configuration Binder source generator * Skip overridden properties in config source generator - include only definitions * Enable shipping Microsoft.Extensions.Configuration.Binder --------- Co-authored-by: Eric StJohn --- .../ConfigurationBindingGenerator.Parser.cs | 6 ++ ...oft.Extensions.Configuration.Binder.csproj | 4 +- .../ConfigurationBinderTests.TestClasses.cs | 59 ++++++++++++++++++ .../tests/Common/ConfigurationBinderTests.cs | 62 +++++++++++++++++++ 4 files changed, 129 insertions(+), 2 deletions(-) diff --git a/src/libraries/Microsoft.Extensions.Configuration.Binder/gen/ConfigurationBindingGenerator.Parser.cs b/src/libraries/Microsoft.Extensions.Configuration.Binder/gen/ConfigurationBindingGenerator.Parser.cs index 2f6a221de90..9522eeb8398 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Binder/gen/ConfigurationBindingGenerator.Parser.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.Binder/gen/ConfigurationBindingGenerator.Parser.cs @@ -654,6 +654,12 @@ private ObjectSpec CreateObjectSpec(TypeParseInfo typeParseInfo) if (member is IPropertySymbol { IsIndexer: false, IsImplicitlyDeclared: false } property && !IsUnsupportedType(property.Type)) { string propertyName = property.Name; + + if (property.IsOverride || properties?.ContainsKey(propertyName) is true) + { + continue; + } + TypeRef propertyTypeRef = EnqueueTransitiveType(typeParseInfo, property.Type, DiagnosticDescriptors.PropertyNotSupported, propertyName); AttributeData? attributeData = property.GetAttributes().FirstOrDefault(a => SymbolEqualityComparer.Default.Equals(a.AttributeClass, _typeSymbols.ConfigurationKeyNameAttribute)); diff --git a/src/libraries/Microsoft.Extensions.Configuration.Binder/src/Microsoft.Extensions.Configuration.Binder.csproj b/src/libraries/Microsoft.Extensions.Configuration.Binder/src/Microsoft.Extensions.Configuration.Binder.csproj index 4d20115e2f7..8ac03929b07 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Binder/src/Microsoft.Extensions.Configuration.Binder.csproj +++ b/src/libraries/Microsoft.Extensions.Configuration.Binder/src/Microsoft.Extensions.Configuration.Binder.csproj @@ -4,8 +4,8 @@ $(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.0;$(NetFrameworkMinimum) true true - false - 1 + true + 2 Provides the functionality to bind an object to data in configuration providers for Microsoft.Extensions.Configuration. This package enables you to represent the configuration data as strongly-typed classes defined in the application code. To bind a configuration, use the Microsoft.Extensions.Configuration.ConfigurationBinder.Get extension method on the IConfiguration object. To use this package, you also need to install a package for the configuration provider, for example, Microsoft.Extensions.Configuration.Json for the JSON provider. diff --git a/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.TestClasses.cs b/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.TestClasses.cs index 9aad9566463..02f3a74f317 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.TestClasses.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.TestClasses.cs @@ -930,5 +930,64 @@ public class SimplePoco public string B { get; set; } } + public class BaseForHiddenMembers + { + public string A { get; set; } + public string B { get; set; } + public TestSettingsEnum E {get; set;} + + public virtual string C { get => CBase; set => CBase = value; } + + public string CBase; + + public virtual string D { get; } + + public virtual string F { get => FBase; set => FBase = value; } + public string FBase; + + + public virtual int X { get => XBase; set => XBase = value; } + public int XBase; + } + + public enum TestSettingsEnum2 + { + // Note - the reflection binder will try to bind to every member + Option1 = TestSettingsEnum.Option1, + Option2 = TestSettingsEnum.Option2, + } + + public class IntermediateDerivedClass : BaseForHiddenMembers + { + public new virtual string D { get => DBase; set => DBase = value; } + public string DBase; + + public override string F { get => "IF"; } + + } + + public class DerivedClassWithHiddenMembers : IntermediateDerivedClass + { + public new string A { get; } = "ADerived"; + public new int B { get; set; } + public new TestSettingsEnum2 E + { + get => (TestSettingsEnum2)base.E; + set => base.E = (TestSettingsEnum)value; + } + + // only override get + public override string C { get => "DC"; } + + // override new only get + public override string D { get => "DD"; } + + // two overrides of only get + public override string F { get => "DF"; } + + // override only set + public override int X { set => base.X = value + 1; } + } + } } diff --git a/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.cs b/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.cs index 21037498613..c45ce4d836b 100644 --- a/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.cs +++ b/src/libraries/Microsoft.Extensions.Configuration.Binder/tests/Common/ConfigurationBinderTests.cs @@ -2486,5 +2486,67 @@ public MockConfigurationRoot(IList providers) : base(pro IConfigurationSection IConfiguration.GetSection(string key) => this[key] is null ? null : new ConfigurationSection(this, key); } + + [Fact] + public void CanBindToClassWithNewProperties() + { + /// the source generator will bind to the most derived property only. + /// the reflection binder will bind the same data to all properties (including hidden). + + var config = TestHelpers.GetConfigurationFromJsonString(""" + { + "A": "AVal", + "B": "5", + "C": "CVal", + "D": "DVal", + "E": "Option2", + "F": "FVal", + "X": "52" + } + """); + var obj = new DerivedClassWithHiddenMembers(); + + config.Bind(obj); + + BaseForHiddenMembers baseObj = obj; + IntermediateDerivedClass intermediateObj = obj; + + Assert.Equal("ADerived", obj.A); +#if BUILDING_SOURCE_GENERATOR_TESTS + // source generator will not set hidden property + Assert.Null(baseObj.A); +#else + // reflection binder will set hidden property + Assert.Equal("AVal", baseObj.A); +#endif + + Assert.Equal(5, obj.B); +#if BUILDING_SOURCE_GENERATOR_TESTS + // source generator will not set hidden property + Assert.Null(baseObj.B); +#else + // reflection binder will set hidden property + Assert.Equal("5", baseObj.B); +#endif + + Assert.Equal(TestSettingsEnum2.Option2, obj.E); + Assert.Equal(TestSettingsEnum.Option2, baseObj.E); + + Assert.Equal("DC", obj.C); + // The setter should still be called, even when only getter is overridden. + Assert.Equal("CVal", obj.CBase); + + // can hide a readonly property with r/w property + Assert.Null(baseObj.D); + Assert.Equal("DD", obj.D); + // The setter should still be called, even when only getter is overridden. + Assert.Equal("DVal", obj.DBase); + + Assert.Equal("DF", obj.F); + Assert.Equal("FVal", obj.FBase); + + Assert.Equal(53, obj.X); + Assert.Equal(53, obj.XBase); + } } } From 97bc9c4a26514021347b0d5173319d7cf18dcda1 Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Wed, 1 May 2024 17:33:53 -0700 Subject: [PATCH 096/151] Update branding to 8.0.6 (#101779) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index f012e409095..bb6bc30992a 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 8.0.5 + 8.0.6 8 0 - 5 + 6 8.0.100 7.0.$([MSBuild]::Add($(PatchVersion),14)) 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) From a0411dfeffcd3c12325290e88eb22828af3f3c63 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 11:08:26 +0200 Subject: [PATCH 097/151] [mono][interp] Resolve virtual method on delegates created by compiled code (#101290) Creating a delegate would normally end up calling into the runtime via ves_icall_mono_delegate_ctor. However, jit/aot backand have a fastpath where the delegate is not fully initialized (relying on the delegate trampoline to resolve the actual method to be called when the delegate is first called). Interp delegate initialization therefore doesn't take place. If this is the case and the delegate method is virtual, we would need to resolve it based on the target object. Co-authored-by: Vlad Brezae --- src/mono/mono/mini/interp/interp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index c7d47719714..620dc135d50 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -4016,6 +4016,8 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause // Not created from interpreted code g_assert (del->method); del_imethod = mono_interp_get_imethod (del->method); + if (del->target && m_method_is_virtual (del->method)) + del_imethod = get_virtual_method (del_imethod, del->target->vtable); del->interp_method = del_imethod; del->interp_invoke_impl = del_imethod; } else { From 37a6d07247d3b69051f6d0997cd42afa30cc129e Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Thu, 2 May 2024 10:35:15 -0700 Subject: [PATCH 098/151] Backport #92375 and #93082 to release/8.0-staging (#101744) * Allow multiple post-build steps and allow templated pre and post-build steps (#92375) * Convert AllConfigurations and NETFX jobs to use the global build templates (#93082) --------- Co-authored-by: Jeremy Koritzinsky --- eng/pipelines/common/global-build-job.yml | 64 ++++-- .../templates/browser-wasm-build-tests.yml | 15 +- .../templates/simple-wasm-build-tests.yml | 15 +- .../common/templates/wasm-build-only.yml | 9 +- .../common/templates/wasm-debugger-tests.yml | 15 +- .../common/templates/wasm-library-tests.yml | 13 +- .../common/templates/wasm-runtime-tests.yml | 9 +- eng/pipelines/coreclr/ci.yml | 3 +- eng/pipelines/coreclr/perf-non-wasm-jobs.yml | 102 ++++----- eng/pipelines/coreclr/perf-wasm-jobs.yml | 14 +- eng/pipelines/coreclr/perf_slow.yml | 19 +- .../runtime-extra-platforms-android.yml | 18 +- ...untime-extra-platforms-androidemulator.yml | 27 +-- .../runtime-extra-platforms-ioslike.yml | 50 ++--- ...ntime-extra-platforms-ioslikesimulator.yml | 39 ++-- .../runtime-extra-platforms-linuxbionic.yml | 9 +- .../runtime-extra-platforms-maccatalyst.yml | 20 +- .../runtime-extra-platforms-other.yml | 30 +-- eng/pipelines/libraries/outerloop-mono.yml | 26 +-- eng/pipelines/libraries/outerloop.yml | 33 +-- .../runtime-android-grpc-client-tests.yml | 11 +- eng/pipelines/runtime-community.yml | 28 +-- eng/pipelines/runtime-linker-tests.yml | 10 +- eng/pipelines/runtime-official.yml | 70 ++++--- eng/pipelines/runtime-sanitized.yml | 39 ++-- eng/pipelines/runtime.yml | 195 ++++++++++-------- eng/pipelines/runtimelab.yml | 27 +-- 27 files changed, 509 insertions(+), 401 deletions(-) diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index a13548956ff..cdda8c99dc5 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -26,8 +26,7 @@ parameters: helixQueues: '' enablePublishTestResults: false testResultsFormat: '' - extraStepsTemplate: '' - extraStepsParameters: {} + postBuildSteps: [] extraVariablesTemplates: [] isManualCodeQLBuild: false preBuildSteps: [] @@ -209,7 +208,28 @@ jobs: - ${{ if ne(parameters.preBuildSteps,'') }}: - ${{ each preBuildStep in parameters.preBuildSteps }}: - - ${{ preBuildStep }} + - ${{ if ne(preBuildStep.template, '') }}: + - template: ${{ preBuildStep.template }} + parameters: + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + buildConfig: ${{ parameters.buildConfig }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} + runtimeVariant: ${{ parameters.runtimeVariant }} + helixQueues: ${{ parameters.helixQueues }} + targetRid: ${{ parameters.targetRid }} + nameSuffix: ${{ parameters.nameSuffix }} + platform: ${{ parameters.platform }} + pgoType: ${{ parameters.pgoType }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + ${{ if ne(preBuildStep.forwardedParameters, '') }}: + ${{ each parameter in preBuildStep.forwardedParameters }}: + ${{ parameter }}: ${{ parameters[parameter] }} + ${{ if ne(preBuildStep.parameters, '') }}: + ${{ insert }}: ${{ preBuildStep.parameters }} + - ${{ else }}: + - ${{ preBuildStep }} # Build - ${{ if eq(parameters.isSourceBuild, false) }}: @@ -235,21 +255,29 @@ jobs: condition: always() # If intended to send extra steps after regular build add them here. - - ${{ if ne(parameters.extraStepsTemplate, '') }}: - - template: ${{ parameters.extraStepsTemplate }} - parameters: - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - runtimeFlavor: ${{ parameters.runtimeFlavor }} - runtimeVariant: ${{ parameters.runtimeVariant }} - helixQueues: ${{ parameters.helixQueues }} - targetRid: ${{ parameters.targetRid }} - nameSuffix: ${{ parameters.nameSuffix }} - platform: ${{ parameters.platform }} - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - ${{ insert }}: ${{ parameters.extraStepsParameters }} + - ${{ if ne(parameters.postBuildSteps,'') }}: + - ${{ each postBuildStep in parameters.postBuildSteps }}: + - ${{ if ne(postBuildStep.template, '') }}: + - template: ${{ postBuildStep.template }} + parameters: + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + archType: ${{ parameters.archType }} + buildConfig: ${{ parameters.buildConfig }} + runtimeFlavor: ${{ parameters.runtimeFlavor }} + runtimeVariant: ${{ parameters.runtimeVariant }} + helixQueues: ${{ parameters.helixQueues }} + targetRid: ${{ parameters.targetRid }} + nameSuffix: ${{ parameters.nameSuffix }} + platform: ${{ parameters.platform }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} + ${{ if ne(postBuildStep.forwardedParameters, '') }}: + ${{ each parameter in postBuildStep.forwardedParameters }}: + ${{ parameter }}: ${{ parameters[parameter] }} + ${{ if ne(postBuildStep.parameters, '') }}: + ${{ insert }}: ${{ postBuildStep.parameters }} + - ${{ else }}: + - ${{ postBuildStep }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log diff --git a/eng/pipelines/common/templates/browser-wasm-build-tests.yml b/eng/pipelines/common/templates/browser-wasm-build-tests.yml index 742ad88de1c..29a3f854724 100644 --- a/eng/pipelines/common/templates/browser-wasm-build-tests.yml +++ b/eng/pipelines/common/templates/browser-wasm-build-tests.yml @@ -117,10 +117,11 @@ jobs: eq(variables['isDefaultPipeline'], variables['shouldRunWasmBuildTestsOnDefaultPipeline'])) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs) - extraHelixArguments: /p:BrowserHost=$(_hostedOs) - scenarios: - - buildwasmapps + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs) + extraHelixArguments: /p:BrowserHost=$(_hostedOs) + scenarios: + - buildwasmapps diff --git a/eng/pipelines/common/templates/simple-wasm-build-tests.yml b/eng/pipelines/common/templates/simple-wasm-build-tests.yml index dcba5522f44..7a593f4f9be 100644 --- a/eng/pipelines/common/templates/simple-wasm-build-tests.yml +++ b/eng/pipelines/common/templates/simple-wasm-build-tests.yml @@ -41,11 +41,12 @@ jobs: eq(variables['alwaysRunVar'], true), eq(variables['isDefaultPipeline'], variables['shouldRunWasmBuildTestsOnDefaultPipeline'])) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs) - extraHelixArguments: /p:BrowserHost=$(_hostedOs) - scenarios: - - buildwasmapps + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig)_$(_hostedOs) + extraHelixArguments: /p:BrowserHost=$(_hostedOs) + scenarios: + - buildwasmapps diff --git a/eng/pipelines/common/templates/wasm-build-only.yml b/eng/pipelines/common/templates/wasm-build-only.yml index 9e9b0cb332c..4f2588e4985 100644 --- a/eng/pipelines/common/templates/wasm-build-only.yml +++ b/eng/pipelines/common/templates/wasm-build-only.yml @@ -38,7 +38,8 @@ jobs: buildArgs: -s mono+libs+packs+libs.tests$(workloadSubsetArg) -c $(_BuildConfig) /p:BrowserHost=$(_hostedOs) ${{ parameters.extraBuildArgs }} /p:TestAssemblies=false $(extraBuildArgs) timeoutInMinutes: 120 condition: ${{ parameters.condition }} - extraStepsTemplate: /eng/pipelines/common/wasm-post-build-steps.yml - extraStepsParameters: - publishArtifactsForWorkload: ${{ parameters.publishArtifactsForWorkload }} - publishWBT: ${{ parameters.publishWBT }} + postBuildSteps: + - template: /eng/pipelines/common/wasm-post-build-steps.yml + parameters: + publishArtifactsForWorkload: ${{ parameters.publishArtifactsForWorkload }} + publishWBT: ${{ parameters.publishWBT }} diff --git a/eng/pipelines/common/templates/wasm-debugger-tests.yml b/eng/pipelines/common/templates/wasm-debugger-tests.yml index fd19fe5385c..17c0f415cf1 100644 --- a/eng/pipelines/common/templates/wasm-debugger-tests.yml +++ b/eng/pipelines/common/templates/wasm-debugger-tests.yml @@ -52,10 +52,11 @@ jobs: and( eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines']), eq(${{ parameters.isWasmOnlyBuild }}, ${{ parameters.runOnlyOnWasmOnlyPipelines }}))) - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_${{ parameters.browser }}_$(_BuildConfig) - extraHelixArguments: /p:BrowserHost=$(_hostedOs) /p:_DebuggerHosts=${{ parameters.browser }} - scenarios: - - wasmdebuggertests + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_${{ parameters.browser }}_$(_BuildConfig) + extraHelixArguments: /p:BrowserHost=$(_hostedOs) /p:_DebuggerHosts=${{ parameters.browser }} + scenarios: + - wasmdebuggertests diff --git a/eng/pipelines/common/templates/wasm-library-tests.yml b/eng/pipelines/common/templates/wasm-library-tests.yml index a848e25e1a2..4260e56ba2c 100644 --- a/eng/pipelines/common/templates/wasm-library-tests.yml +++ b/eng/pipelines/common/templates/wasm-library-tests.yml @@ -62,9 +62,10 @@ jobs: eq(variables['alwaysRunVar'], true), eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines'])) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - extraHelixArguments: /p:BrowserHost=$(_hostedOs) $(_wasmRunSmokeTestsOnlyArg) ${{ parameters.extraHelixArgs }} - scenarios: ${{ parameters.scenarios }} + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + extraHelixArguments: /p:BrowserHost=$(_hostedOs) $(_wasmRunSmokeTestsOnlyArg) ${{ parameters.extraHelixArgs }} + scenarios: ${{ parameters.scenarios }} diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml index 43671a546ba..2b006bb2db3 100644 --- a/eng/pipelines/common/templates/wasm-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasm-runtime-tests.yml @@ -42,9 +42,10 @@ jobs: or( eq(variables['alwaysRunVar'], true), eq(variables['isDefaultPipeline'], variables['shouldRunOnDefaultPipelines'])) - extraStepsTemplate: //eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml diff --git a/eng/pipelines/coreclr/ci.yml b/eng/pipelines/coreclr/ci.yml index 17806e94407..afff0abf00b 100644 --- a/eng/pipelines/coreclr/ci.yml +++ b/eng/pipelines/coreclr/ci.yml @@ -174,4 +174,5 @@ extends: jobParameters: buildArgs: -s clr.paltests+clr.paltestlist nameSuffix: PALTests - extraStepsTemplate: /eng/pipelines/coreclr/templates/run-paltests-step.yml + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/run-paltests-step.yml diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index b7f87d7cf89..3e28bb7bcae 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -48,15 +48,16 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=true /p:MonoLibClang="/usr/local/lib/libclang.so.16" /p:AotHostArchitecture=x64 /p:AotHostOS=linux nameSuffix: AOT isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: AOT Mono Artifacts - artifactName: LinuxMonoAOTx64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: AOT Mono Artifacts + artifactName: LinuxMonoAOTx64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono Android scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -70,15 +71,16 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) nameSuffix: AndroidMono isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: Android Mono Artifacts - artifactName: AndroidMonoarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: Android Mono Artifacts + artifactName: AndroidMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono iOS scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -92,15 +94,16 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) nameSuffix: iOSMono isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: iOS Mono Artifacts - artifactName: iOSMonoarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: iOS Mono Artifacts + artifactName: iOSMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build NativeAOT iOS scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -114,15 +117,16 @@ jobs: buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs -c $(_BuildConfig) nameSuffix: iOSNativeAOT isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: iOS NativeAOT Artifacts - artifactName: iOSNativeAOTarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: iOS NativeAOT Artifacts + artifactName: iOSNativeAOTarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono - template: /eng/pipelines/common/platform-matrix.yml @@ -409,9 +413,10 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) nameSuffix: Mono_Packs isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks # build PerfBDN app - template: /eng/pipelines/common/platform-matrix.yml @@ -429,12 +434,13 @@ jobs: isOfficialBuild: false pool: vmImage: 'macos-12' - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: Android BDN App Artifacts - artifactName: PerfBDNAppArm - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: Android BDN App Artifacts + artifactName: PerfBDNAppArm + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz diff --git a/eng/pipelines/coreclr/perf-wasm-jobs.yml b/eng/pipelines/coreclr/perf-wasm-jobs.yml index 7758eac4d12..bbfba488347 100644 --- a/eng/pipelines/coreclr/perf-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-wasm-jobs.yml @@ -25,9 +25,10 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: wasm isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml - extraStepsParameters: - configForBuild: Release + postBuildSteps: + - template: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml + parameters: + configForBuild: Release #run mono wasm microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml @@ -94,9 +95,10 @@ jobs: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: wasm isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml - extraStepsParameters: - configForBuild: Release + postBuildSteps: + - template: /eng/pipelines/coreclr/perf-wasm-prepare-artifacts-steps.yml + parameters: + configForBuild: Release # run mono wasm interpreter (default) microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/perf_slow.yml b/eng/pipelines/coreclr/perf_slow.yml index 91e3ad34f38..d6124364554 100644 --- a/eng/pipelines/coreclr/perf_slow.yml +++ b/eng/pipelines/coreclr/perf_slow.yml @@ -143,15 +143,16 @@ extends: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true /p:BuildMonoAOTCrossCompiler=true /p:MonoLibClang="/usr/local/lib/libclang.so.16" /p:AotHostArchitecture=arm64 /p:AotHostOS=linux nameSuffix: AOT isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: AOT Mono Artifacts - artifactName: LinuxMonoAOTarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + postBuildSteps: + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: AOT Mono Artifacts + artifactName: LinuxMonoAOTarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-android.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-android.yml index 23c57b87fe5..9fd1769fe18 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-android.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-android.yml @@ -44,10 +44,11 @@ jobs: # Turn off the testing for now, until https://github.com/dotnet/runtime/issues/60128 gets resolved # ${{ if eq(variables['isRollingBuild'], true) }}: # # extra steps, run tests - # extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - # extraStepsParameters: - # creator: dotnet-bot - # testRunNamePrefixSuffix: Mono_$(_BuildConfig) + # postBuildSteps: + # - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + # parameters: + # creator: dotnet-bot + # testRunNamePrefixSuffix: Mono_$(_BuildConfig) # extraVariablesTemplates: # - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -77,7 +78,8 @@ jobs: buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:EnableAdditionalTimezoneChecks=true timeoutInMinutes: 480 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml index 65a890976e7..a114b1b744a 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-androidemulator.yml @@ -40,10 +40,11 @@ jobs: buildArgs: -s mono+libs -c $(_BuildConfig) timeoutInMinutes: 240 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -78,10 +79,11 @@ jobs: buildArgs: -s mono+libs -c $(_BuildConfig) timeoutInMinutes: 240 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -111,7 +113,8 @@ jobs: buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslike.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslike.yml index 84477d2c25a..ef0425042b5 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslike.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslike.yml @@ -38,11 +38,12 @@ jobs: buildArgs: -s mono+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:DevTeamProvisioning=- /p:RunAOTCompilation=true $(_runSmokeTestsOnlyArg) /p:BuildTestsOnHelix=true /p:EnableAdditionalTimezoneChecks=true /p:UsePortableRuntimePack=true /p:BuildDarwinFrameworks=true /p:IsManualOrRollingBuild=true timeoutInMinutes: 480 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true # # iOS/tvOS devices @@ -80,14 +81,15 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: testGroup: innerloop - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - compileOnHelix: true - interpreter: true - testBuildArgs: /p:ArchiveTests=true /p:DevTeamProvisioning=- /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildTestsOnHelix=true - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + compileOnHelix: true + interpreter: true + testBuildArgs: /p:ArchiveTests=true /p:DevTeamProvisioning=- /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildTestsOnHelix=true + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true # # iOS/tvOS devices @@ -116,11 +118,12 @@ jobs: buildArgs: --cross -s clr.alljits+clr.tools+clr.nativeaotruntime+clr.nativeaotlibs+libs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:RunSmokeTestsOnly=true /p:DevTeamProvisioning=- /p:BuildTestsOnHelix=true /p:UseNativeAOTRuntime=true /p:RunAOTCompilation=false /p:ContinuousIntegrationBuild=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true # # Build the whole product using NativeAOT for iOS/tvOS and run runtime tests with iOS/tvOS devices @@ -151,8 +154,9 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: testGroup: innerloop - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: tree nativeaot/SmokeTests /p:BuildNativeAOTRuntimePack=true - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testBuildArgs: tree nativeaot/SmokeTests /p:BuildNativeAOTRuntimePack=true + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslikesimulator.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslikesimulator.yml index c7bcfc15e13..b11b4be72ed 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslikesimulator.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-ioslikesimulator.yml @@ -40,11 +40,12 @@ jobs: buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildDarwinFrameworks=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - interpreter: true - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + interpreter: true + testRunNamePrefixSuffix: Mono_$(_BuildConfig) # # Build the whole product using Mono for iOSSimulator/tvOSSimulator and run runtime tests with iOS/tvOS simulators @@ -84,14 +85,15 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: testGroup: innerloop - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - compileOnHelix: true - interpreter: true - testBuildArgs: /p:ArchiveTests=true /p:DevTeamProvisioning=- /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildTestsOnHelix=true - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + compileOnHelix: true + interpreter: true + testBuildArgs: /p:ArchiveTests=true /p:DevTeamProvisioning=- /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildTestsOnHelix=true + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true # # Build the whole product using Native AOT for iOSSimulator/tvOSSimulator and run runtime tests with iOS/tvOS simulators @@ -131,8 +133,9 @@ jobs: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: testGroup: innerloop - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: tree nativeaot/SmokeTests /p:BuildNativeAOTRuntimePack=true - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testBuildArgs: tree nativeaot/SmokeTests /p:BuildNativeAOTRuntimePack=true + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-linuxbionic.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-linuxbionic.yml index 352eabe9d22..7eb5fa47f80 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-linuxbionic.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-linuxbionic.yml @@ -42,7 +42,8 @@ jobs: buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true timeoutInMinutes: 480 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig)_LinuxBionic + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig)_LinuxBionic diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-maccatalyst.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-maccatalyst.yml index 736bf0516df..936fe60bb48 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-maccatalyst.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-maccatalyst.yml @@ -37,10 +37,11 @@ jobs: buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:DevTeamProvisioning=adhoc /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildDarwinFrameworks=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) # # MacCatalyst interp - requires AOT Compilation and Interp flags @@ -70,8 +71,9 @@ jobs: buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true $(_runSmokeTestsOnlyArg) /p:DevTeamProvisioning=adhoc /p:RunAOTCompilation=true /p:MonoForceInterpreter=true /p:BuildDarwinFrameworks=true /p:EnableAppSandbox=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - interpreter: true - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + interpreter: true + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml index c279c318e34..ee481119ad9 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml @@ -279,15 +279,16 @@ jobs: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # Build the whole product using Mono and run runtime tests @@ -380,11 +381,12 @@ jobs: eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - llvmAotStepContainer: linux_x64_llvmaot - testRunNamePrefixSuffix: Mono_Release + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + llvmAotStepContainer: linux_x64_llvmaot + testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml diff --git a/eng/pipelines/libraries/outerloop-mono.yml b/eng/pipelines/libraries/outerloop-mono.yml index e15fc35f9d4..34b1af3c71b 100644 --- a/eng/pipelines/libraries/outerloop-mono.yml +++ b/eng/pipelines/libraries/outerloop-mono.yml @@ -39,11 +39,12 @@ extends: timeoutInMinutes: 180 includeAllPlatforms: ${{ variables['isRollingBuild'] }} # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - testScope: outerloop - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + testScope: outerloop + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -60,10 +61,11 @@ extends: timeoutInMinutes: 180 includeAllPlatforms: ${{ variables['isRollingBuild'] }} # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - scenarios: - - normal - testScope: outerloop - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + scenarios: + - normal + testScope: outerloop + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/libraries/outerloop.yml b/eng/pipelines/libraries/outerloop.yml index 121d405fc7c..597f298c37a 100644 --- a/eng/pipelines/libraries/outerloop.yml +++ b/eng/pipelines/libraries/outerloop.yml @@ -45,11 +45,12 @@ extends: timeoutInMinutes: 180 includeAllPlatforms: ${{ variables['isRollingBuild'] }} # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - testScope: outerloop - creator: dotnet-bot - testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + testScope: outerloop + creator: dotnet-bot + testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) - ${{ if eq(variables['isRollingBuild'], false) }}: - template: /eng/pipelines/common/platform-matrix.yml @@ -73,11 +74,12 @@ extends: timeoutInMinutes: 180 includeAllPlatforms: ${{ variables['isRollingBuild'] }} # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - testScope: outerloop - creator: dotnet-bot - testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + testScope: outerloop + creator: dotnet-bot + testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) - ${{ if eq(variables['includeWindowsOuterloop'], true) }}: - template: /eng/pipelines/common/platform-matrix.yml @@ -97,8 +99,9 @@ extends: timeoutInMinutes: 180 includeAllPlatforms: ${{ variables['isRollingBuild'] }} # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - testScope: outerloop - creator: dotnet-bot - extraHelixArguments: /p:BuildTargetFramework=net48 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + testScope: outerloop + creator: dotnet-bot + extraHelixArguments: /p:BuildTargetFramework=net48 diff --git a/eng/pipelines/runtime-android-grpc-client-tests.yml b/eng/pipelines/runtime-android-grpc-client-tests.yml index 00e51f766d9..707b1d78ebc 100644 --- a/eng/pipelines/runtime-android-grpc-client-tests.yml +++ b/eng/pipelines/runtime-android-grpc-client-tests.yml @@ -43,8 +43,9 @@ extends: buildArgs: -s mono+libs+host+packs+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true /p:RunGrpcTestsOnly=true /p:BuildGrpcServerDockerImage=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - extraHelixArguments: /p:RunGrpcTestsOnly=true /p:BuildGrpcServerDockerImage=true - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + extraHelixArguments: /p:RunGrpcTestsOnly=true /p:BuildGrpcServerDockerImage=true + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/runtime-community.yml b/eng/pipelines/runtime-community.yml index bab086f75c2..a91388e244b 100644 --- a/eng/pipelines/runtime-community.yml +++ b/eng/pipelines/runtime-community.yml @@ -71,15 +71,16 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # Build the whole product using Mono @@ -138,7 +139,8 @@ extends: eq(variables['isRollingBuild'], true)) ${{ if eq(variables['isRollingBuild'], true) }}: # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) diff --git a/eng/pipelines/runtime-linker-tests.yml b/eng/pipelines/runtime-linker-tests.yml index 9db2062a953..1cb0f8428ef 100644 --- a/eng/pipelines/runtime-linker-tests.yml +++ b/eng/pipelines/runtime-linker-tests.yml @@ -103,7 +103,8 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) buildArgs: -s clr+libs+tools.illink -c $(_BuildConfig) - extraStepsTemplate: /eng/pipelines/libraries/execute-trimming-tests-steps.yml + postBuildSteps: + - template: /eng/pipelines/libraries/execute-trimming-tests-steps.yml # # Build Release config vertical for Browser-wasm @@ -126,6 +127,7 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_wasm_specific_except_wbt_dbg.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_NET_ILLink_Tasks'], true)) - extraStepsTemplate: /eng/pipelines/libraries/execute-trimming-tests-steps.yml - extraStepsParameters: - extraTestArgs: '/p:WasmBuildNative=false' + postBuildSteps: + - template: /eng/pipelines/libraries/execute-trimming-tests-steps.yml + parameters: + extraTestArgs: '/p:WasmBuildNative=false' diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 34658018fca..3138f18ebd7 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -124,9 +124,10 @@ extends: buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true nameSuffix: AllSubsets_NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: NativeAOTRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: NativeAOTRuntimePacks # # Build Mono runtime packs @@ -166,9 +167,10 @@ extends: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false nameSuffix: AllSubsets_Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -182,9 +184,10 @@ extends: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: AllSubsets_Mono isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -198,9 +201,10 @@ extends: nameSuffix: AllSubsets_Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks # Build Mono AOT offset headers once, for consumption elsewhere # @@ -242,9 +246,10 @@ extends: - android - browser isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -265,9 +270,10 @@ extends: - android - browser isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -295,9 +301,10 @@ extends: - ios - maccatalyst isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks # # Build Mono LLVM runtime packs @@ -325,9 +332,10 @@ extends: nameSuffix: AllSubsets_Mono_LLVMJIT runtimeVariant: LLVMJIT isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks #LLVMAOT - jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release @@ -338,9 +346,10 @@ extends: nameSuffix: AllSubsets_Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: MonoRuntimePacks + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: MonoRuntimePacks # # Build libraries using live CoreLib from CoreCLR @@ -395,9 +404,10 @@ extends: - SourceBuild_linux_x64 jobParameters: nameSuffix: PortableSourceBuild - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: SourceBuildPackages + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: SourceBuildPackages timeoutInMinutes: 95 # diff --git a/eng/pipelines/runtime-sanitized.yml b/eng/pipelines/runtime-sanitized.yml index 3bc49fec690..5db421762ed 100644 --- a/eng/pipelines/runtime-sanitized.yml +++ b/eng/pipelines/runtime-sanitized.yml @@ -38,13 +38,14 @@ extends: buildArgs: -s clr+libs -c $(_BuildConfig) $(_nativeSanitizersArg) timeoutInMinutes: 300 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) - scenarios: - - normal - - no_tiered_compilation + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: CoreCLR_$(_BuildConfig) + scenarios: + - normal + - no_tiered_compilation extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: @@ -72,12 +73,13 @@ extends: buildArgs: -s clr+libs+libs.tests -c $(_BuildConfig) -rc Checked $(_nativeSanitizersArg) /p:ArchiveTests=true timeoutInMinutes: 180 # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Libraries_$(_BuildConfig) - scenarios: - - normal + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Libraries_$(_BuildConfig) + scenarios: + - normal # # NativeAOT release build and smoke tests with AddressSanitizer @@ -98,11 +100,12 @@ extends: timeoutInMinutes: 120 nameSuffix: NativeAOT buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release $(_nativeSanitizersArg) - extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: nativeaot tree nativeaot - liveLibrariesBuildConfig: Release + postBuildSteps: + - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml + parameters: + creator: dotnet-bot + testBuildArgs: nativeaot tree nativeaot + liveLibrariesBuildConfig: Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index f2657982aee..c83a76993e6 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -103,9 +103,10 @@ extends: testGroup: innerloop nameSuffix: Native_GCC buildArgs: -s clr.native+libs.native+mono+host.native -c $(_BuildConfig) -gcc - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml - extraStepsParameters: - testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests.yml + parameters: + testBuildArgs: skipmanaged skipgeneratelayout skiprestorepackages -gcc condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), @@ -249,12 +250,13 @@ extends: timeoutInMinutes: 120 nameSuffix: NativeAOT buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release - extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: nativeaot tree nativeaot - liveLibrariesBuildConfig: Release - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml + parameters: + creator: dotnet-bot + testBuildArgs: nativeaot tree nativeaot + liveLibrariesBuildConfig: Release + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: @@ -286,12 +288,13 @@ extends: timeoutInMinutes: 180 nameSuffix: NativeAOT buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release - extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: 'nativeaot tree ";nativeaot;Loader;Interop;tracing;" /p:BuildNativeAotFrameworkObjects=true' - liveLibrariesBuildConfig: Release - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml + parameters: + creator: dotnet-bot + testBuildArgs: 'nativeaot tree ";nativeaot;Loader;Interop;tracing;" /p:BuildNativeAotFrameworkObjects=true' + liveLibrariesBuildConfig: Release + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: @@ -329,12 +332,13 @@ extends: timeoutInMinutes: 120 nameSuffix: NativeAOT buildArgs: -s clr.aot+host.native+libs+tools.illink -c $(_BuildConfig) -rc $(_BuildConfig) -lc Release -hc Release - extraStepsTemplate: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml - extraStepsParameters: - creator: dotnet-bot - testBuildArgs: 'nativeaot tree ";nativeaot;tracing/eventpipe/providervalidation;"' - liveLibrariesBuildConfig: Release - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml + parameters: + creator: dotnet-bot + testBuildArgs: 'nativeaot tree ";nativeaot;tracing/eventpipe/providervalidation;"' + liveLibrariesBuildConfig: Release + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml parameters: @@ -366,10 +370,11 @@ extends: buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true /p:ArchiveTests=true timeoutInMinutes: 240 # Doesn't actually take long, but we've seen the ARM64 Helix queue often get backlogged for 2+ hours # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -627,15 +632,16 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # iOS/tvOS devices - Full AOT + AggressiveTrimming to reduce size @@ -668,16 +674,17 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # iOS/tvOS devices @@ -710,16 +717,17 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) - extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['coreclrContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NativeAOT_$(_BuildConfig) + extraHelixArguments: /p:NeedsToBuildAppsOnHelix=true + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['coreclrContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # MacCatalyst interp - requires AOT Compilation and Interp flags @@ -753,15 +761,16 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), eq(variables['isRollingBuild'], true)) # extra steps, run tests - extraStepsTemplate: /eng/pipelines/libraries/helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_$(_BuildConfig) + condition: >- + or( + eq(variables['librariesContainsChange'], true), + eq(variables['monoContainsChange'], true), + eq(variables['isRollingBuild'], true)) # # Build Mono and Installer on LLVMJIT mode @@ -1073,32 +1082,45 @@ extends: condition: >- eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true) + # + # Build and test libraries for .NET Framework + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - windows_x86 helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml jobParameters: framework: net48 - runTests: true - testScope: innerloop + buildArgs: -s tools+libs+libs.tests -framework net48 -c $(_BuildConfig) -testscope innerloop /p:ArchiveTests=true + nameSuffix: Libraries_NET48 + timeoutInMinutes: 150 + postBuildSteps: + - template: /eng/pipelines/libraries/helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: NET48_$(_BuildConfig) + extraHelixArguments: /p:BuildTargetFramework=net48 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build and test libraries AllConfigurations + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} platforms: - windows_x64 jobParameters: - framework: allConfigurations - runTests: true - useHelix: false + buildArgs: -test -s tools+libs+libs.tests -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations + timeoutInMinutes: 150 condition: >- or( eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), @@ -1294,10 +1316,11 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_Release + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -1329,10 +1352,11 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_Release + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # @@ -1366,11 +1390,12 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), eq(variables['isRollingBuild'], true)) - extraStepsTemplate: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - extraStepsParameters: - creator: dotnet-bot - llvmAotStepContainer: linux_x64_llvmaot - testRunNamePrefixSuffix: Mono_Release + postBuildSteps: + - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml + parameters: + creator: dotnet-bot + llvmAotStepContainer: linux_x64_llvmaot + testRunNamePrefixSuffix: Mono_Release extraVariablesTemplates: - template: /eng/pipelines/common/templates/runtimes/test-variables.yml @@ -1565,8 +1590,6 @@ extends: - SourceBuild_centos8_x64 jobParameters: nameSuffix: centos8SourceBuild - extraStepsParameters: - name: SourceBuildPackages timeoutInMinutes: 95 condition: eq(variables['isRollingBuild'], true) @@ -1579,7 +1602,5 @@ extends: - SourceBuild_banana24_x64 jobParameters: nameSuffix: banana24SourceBuild - extraStepsParameters: - name: SourceBuildPackages timeoutInMinutes: 95 condition: eq(variables['isRollingBuild'], true) diff --git a/eng/pipelines/runtimelab.yml b/eng/pipelines/runtimelab.yml index a5c4e033433..7c34126757d 100644 --- a/eng/pipelines/runtimelab.yml +++ b/eng/pipelines/runtimelab.yml @@ -65,9 +65,10 @@ extends: timeoutInMinutes: 100 testGroup: innerloop buildArgs: -s clr+libs+host+packs -c debug -runtimeConfiguration Checked - extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - extraStepsParameters: - uploadRuntimeTests: true + postBuildSteps: + - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml + parameters: + uploadRuntimeTests: true # # Build with Release config and Release runtimeConfiguration @@ -83,10 +84,11 @@ extends: timeoutInMinutes: 100 isOfficialBuild: ${{ variables.isOfficialBuild }} testGroup: innerloop - extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - extraStepsParameters: - uploadLibrariesTests: ${{ eq(variables.isOfficialBuild, false) }} - uploadIntermediateArtifacts: false + postBuildSteps: + - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml + parameters: + uploadLibrariesTests: ${{ eq(variables.isOfficialBuild, false) }} + uploadIntermediateArtifacts: false ${{ if eq(variables.isOfficialBuild, false) }}: buildArgs: -s clr+libs+libs.tests+host+packs -c $(_BuildConfig) /p:ArchiveTests=true ${{ if eq(variables.isOfficialBuild, true) }}: @@ -107,11 +109,12 @@ extends: nameSuffix: AllConfigurations buildArgs: -s libs -c $(_BuildConfig) -allConfigurations ${{ if eq(variables.isOfficialBuild, true) }}: - extraStepsTemplate: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml - extraStepsParameters: - uploadIntermediateArtifacts: true - isOfficialBuild: true - librariesBinArtifactName: libraries_bin_official_allconfigurations + postBuildSteps: + - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml + parameters: + uploadIntermediateArtifacts: true + isOfficialBuild: true + librariesBinArtifactName: libraries_bin_official_allconfigurations # Installer official builds need to build installers and need the libraries all configurations build - ${{ if eq(variables.isOfficialBuild, true) }}: From 144fc6112ca791e9e24c6d28cfc22a123d5c970e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 11:27:58 -0700 Subject: [PATCH 099/151] Remove dead code in PGO scenarios. (#101649) Co-authored-by: Aaron R Robinson --- src/coreclr/vm/jitinterface.cpp | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 1cc6c37e09c..67e127bf91e 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -11881,23 +11881,6 @@ HRESULT CEEJitInfo::allocPgoInstrumentationBySchema( JIT_TO_EE_TRANSITION(); - // We need to know the code size. Typically we can get the code size - // from m_ILHeader. For dynamic methods, m_ILHeader will be NULL, so - // for that case we need to use DynamicResolver to get the code size. - - unsigned codeSize = 0; - if (m_pMethodBeingCompiled->IsDynamicMethod()) - { - unsigned stackSize, ehSize; - CorInfoOptions options; - DynamicResolver * pResolver = m_pMethodBeingCompiled->AsDynamicMethodDesc()->GetResolver(); - pResolver->GetCodeInfo(&codeSize, &stackSize, &options, &ehSize); - } - else - { - codeSize = m_ILHeader->GetCodeSize(); - } - #ifdef FEATURE_PGO hr = PgoManager::allocPgoInstrumentationBySchema(m_pMethodBeingCompiled, pSchema, countSchemaItems, pInstrumentationData); #else From 13a33f146f937737050dd659b0b7d9c1945682ef Mon Sep 17 00:00:00 2001 From: Egor Bogatov Date: Thu, 2 May 2024 20:31:17 +0200 Subject: [PATCH 100/151] JIT: Track sideness of arrOp in GetCheckedBoundArithInfo (#100968) --- src/coreclr/jit/optcse.cpp | 7 ++++-- src/coreclr/jit/valuenum.cpp | 14 ++++++----- src/coreclr/jit/valuenum.h | 3 ++- .../JitBlue/Runtime_100809/Runtime_100809.cs | 23 +++++++++++++++++++ .../Runtime_100809/Runtime_100809.csproj | 5 ++++ 5 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.cs create mode 100644 src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.csproj diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index 4e7fc81df64..505352250f4 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -3066,8 +3066,11 @@ class CSE_Heuristic assert(vnStore->IsVNCompareCheckedBoundArith(oldCmpVN)); vnStore->GetCompareCheckedBoundArithInfo(oldCmpVN, &info); - newCmpArgVN = vnStore->VNForFunc(vnStore->TypeOfVN(info.arrOp), (VNFunc)info.arrOper, - info.arrOp, theConservativeVN); + ValueNum arrOp1 = info.arrOpLHS ? info.arrOp : theConservativeVN; + ValueNum arrOp2 = info.arrOpLHS ? theConservativeVN : info.arrOp; + + newCmpArgVN = + vnStore->VNForFunc(vnStore->TypeOfVN(info.arrOp), (VNFunc)info.arrOper, arrOp1, arrOp2); } ValueNum newCmpVN = vnStore->VNForFunc(vnStore->TypeOfVN(oldCmpVN), (VNFunc)info.cmpOper, info.cmpOp, newCmpArgVN); diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index fa60633afbc..fbc80330eb9 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -6576,15 +6576,17 @@ void ValueNumStore::GetCheckedBoundArithInfo(ValueNum vn, CompareCheckedBoundAri bool isOp1CheckedBound = IsVNCheckedBound(funcArith.m_args[1]); if (isOp1CheckedBound) { - info->arrOper = funcArith.m_func; - info->arrOp = funcArith.m_args[0]; - info->vnBound = funcArith.m_args[1]; + info->arrOper = funcArith.m_func; + info->arrOp = funcArith.m_args[0]; + info->vnBound = funcArith.m_args[1]; + info->arrOpLHS = true; } else { - info->arrOper = funcArith.m_func; - info->arrOp = funcArith.m_args[1]; - info->vnBound = funcArith.m_args[0]; + info->arrOper = funcArith.m_func; + info->arrOp = funcArith.m_args[1]; + info->vnBound = funcArith.m_args[0]; + info->arrOpLHS = false; } } diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index 04fed7bfbc1..af6ba529014 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -920,9 +920,10 @@ class ValueNumStore ValueNum vnBound; unsigned arrOper; ValueNum arrOp; + bool arrOpLHS; // arrOp is on the left side of cmpOp expression unsigned cmpOper; ValueNum cmpOp; - CompareCheckedBoundArithInfo() : vnBound(NoVN), arrOper(GT_NONE), arrOp(NoVN), cmpOper(GT_NONE), cmpOp(NoVN) + CompareCheckedBoundArithInfo() : vnBound(NoVN), arrOper(GT_NONE), arrOp(NoVN), arrOpLHS(false), cmpOper(GT_NONE), cmpOp(NoVN) { } #ifdef DEBUG diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.cs b/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.cs new file mode 100644 index 00000000000..e7c94f11ac2 --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +public static class Runtime_100809 +{ + [Fact] + public static int TestEntryPoint() + { + return AlwaysFalse(96) ? -1 : 100; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool AlwaysFalse(int x) + { + var result = new byte[x]; + int count = result.Length - 2; + return (x < 0 || result.Length - count < 0); + } +} \ No newline at end of file diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.csproj b/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.csproj new file mode 100644 index 00000000000..197767e2c4e --- /dev/null +++ b/src/tests/JIT/Regression/JitBlue/Runtime_100809/Runtime_100809.csproj @@ -0,0 +1,5 @@ + + + + + From 40804203a22be83c5a13bb85230718257dbb7ad8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 12:36:32 -0700 Subject: [PATCH 101/151] Revert "disable optimizations for PopCount (#99796)" (#101245) This reverts commit aee49579769188d0ff7cf3ca872d2126e5bb3c70. Co-authored-by: Manish Godse <61718172+mangod9@users.noreply.github.com> --- src/coreclr/jit/utils.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index 4c9fe6479ec..2e1c0a52a3d 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -3192,11 +3192,6 @@ uint32_t BitOperations::Log2(uint64_t value) // Return Value: // The population count (number of bits set) of value // -#if defined(_MSC_VER) -// Disable optimizations for PopCount to avoid the compiler from generating intrinsics -// not supported on all platforms. -#pragma optimize("", off) -#endif // _MSC_VER uint32_t BitOperations::PopCount(uint32_t value) { #if defined(_MSC_VER) @@ -3249,9 +3244,6 @@ uint32_t BitOperations::PopCount(uint64_t value) return static_cast(result); #endif } -#if defined(_MSC_VER) -#pragma optimize("", on) -#endif // _MSC_VER //------------------------------------------------------------------------ // BitOperations::ReverseBits: Reverses the bits in an integer value From 70f51120fde46628c497d383cecfa3992682dc8e Mon Sep 17 00:00:00 2001 From: Kevin Jones Date: Thu, 2 May 2024 16:54:30 -0400 Subject: [PATCH 102/151] Add cryptographic operation counts to prevent process crashes --- .../src/Resources/Strings.resx | 3 + .../src/System.Security.Cryptography.csproj | 2 + .../Cryptography/ConcurrencyBlock.NoOp.cs | 25 +++++++ .../Security/Cryptography/ConcurrencyBlock.cs | 40 +++++++++++ .../HashProviderDispenser.OpenSsl.cs | 66 +++++++++++++------ 5 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.NoOp.cs create mode 100644 src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.cs diff --git a/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx b/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx index 1bd529a087a..4bcfdcfd345 100644 --- a/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx +++ b/src/libraries/System.Security.Cryptography/src/Resources/Strings.resx @@ -273,6 +273,9 @@ The specified CipherMode '{0}' is not supported. + + Concurrent operations from multiple threads on this type are not supported. + This key is for algorithm '{0}'. Expected '{1}'. diff --git a/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj b/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj index 691235d0de2..591ae21cfd7 100644 --- a/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj +++ b/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj @@ -817,6 +817,7 @@ + @@ -977,6 +978,7 @@ + diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.NoOp.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.NoOp.cs new file mode 100644 index 00000000000..18736cacfc2 --- /dev/null +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.NoOp.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; + +namespace System.Security.Cryptography +{ + internal struct ConcurrencyBlock + { + internal static Scope Enter(ref ConcurrencyBlock block) + { + _ = block; + return default; + } + + internal ref struct Scope + { +#pragma warning disable CA1822 // Member can be marked static + internal void Dispose() + { + } +#pragma warning restore CA1822 + } + } +} diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.cs new file mode 100644 index 00000000000..c0eafe849da --- /dev/null +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ConcurrencyBlock.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; + +namespace System.Security.Cryptography +{ + internal struct ConcurrencyBlock + { + private int _count; + + internal static Scope Enter(ref ConcurrencyBlock block) + { + int count = Interlocked.Increment(ref block._count); + + if (count != 1) + { + Interlocked.Decrement(ref block._count); + throw new CryptographicException(SR.Cryptography_ConcurrentUseNotSupported); + } + + return new Scope(ref block._count); + } + + internal ref struct Scope + { + private ref int _parentCount; + + internal Scope(ref int parentCount) + { + _parentCount = ref parentCount; + } + + internal void Dispose() + { + Interlocked.Decrement(ref _parentCount); + } + } + } +} diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderDispenser.OpenSsl.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderDispenser.OpenSsl.cs index a048496fc3f..ef15a1736b6 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderDispenser.OpenSsl.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderDispenser.OpenSsl.cs @@ -102,6 +102,7 @@ private sealed class EvpHashProvider : HashProvider { private readonly LiteHash _liteHash; private bool _running; + private ConcurrencyBlock _block; public EvpHashProvider(string hashAlgorithmId) { @@ -110,21 +111,30 @@ public EvpHashProvider(string hashAlgorithmId) public override void AppendHashData(ReadOnlySpan data) { - _liteHash.Append(data); - _running = true; + using (ConcurrencyBlock.Enter(ref _block)) + { + _liteHash.Append(data); + _running = true; + } } public override int FinalizeHashAndReset(Span destination) { - int written = _liteHash.Finalize(destination); - _liteHash.Reset(); - _running = false; - return written; + using (ConcurrencyBlock.Enter(ref _block)) + { + int written = _liteHash.Finalize(destination); + _liteHash.Reset(); + _running = false; + return written; + } } public override int GetCurrentHash(Span destination) { - return _liteHash.Current(destination); + using (ConcurrencyBlock.Enter(ref _block)) + { + return _liteHash.Current(destination); + } } public override int HashSizeInBytes => _liteHash.HashSizeInBytes; @@ -139,10 +149,13 @@ public override void Dispose(bool disposing) public override void Reset() { - if (_running) + using (ConcurrencyBlock.Enter(ref _block)) { - _liteHash.Reset(); - _running = false; + if (_running) + { + _liteHash.Reset(); + _running = false; + } } } } @@ -151,6 +164,7 @@ private sealed class HmacHashProvider : HashProvider { private readonly LiteHmac _liteHmac; private bool _running; + private ConcurrencyBlock _block; public HmacHashProvider(string hashAlgorithmId, ReadOnlySpan key) { @@ -159,21 +173,30 @@ public HmacHashProvider(string hashAlgorithmId, ReadOnlySpan key) public override void AppendHashData(ReadOnlySpan data) { - _liteHmac.Append(data); - _running = true; + using (ConcurrencyBlock.Enter(ref _block)) + { + _liteHmac.Append(data); + _running = true; + } } public override int FinalizeHashAndReset(Span destination) { - int written = _liteHmac.Finalize(destination); - _liteHmac.Reset(); - _running = false; - return written; + using (ConcurrencyBlock.Enter(ref _block)) + { + int written = _liteHmac.Finalize(destination); + _liteHmac.Reset(); + _running = false; + return written; + } } public override int GetCurrentHash(Span destination) { - return _liteHmac.Current(destination); + using (ConcurrencyBlock.Enter(ref _block)) + { + return _liteHmac.Current(destination); + } } public override int HashSizeInBytes => _liteHmac.HashSizeInBytes; @@ -188,10 +211,13 @@ public override void Dispose(bool disposing) public override void Reset() { - if (_running) + using (ConcurrencyBlock.Enter(ref _block)) { - _liteHmac.Reset(); - _running = false; + if (_running) + { + _liteHmac.Reset(); + _running = false; + } } } } From 4021666c9a7e528dc4897955f76c44e7e16dc7e9 Mon Sep 17 00:00:00 2001 From: Michael Simons Date: Thu, 2 May 2024 16:16:36 -0500 Subject: [PATCH 103/151] Revert "Enable repackaging of NETStandard 2.1 Targeting Pack (#101518)" (#101812) This reverts commit c67a2e4216e156f4129638d71ea99a6a731106ed. --- eng/Subsets.props | 1 - eng/Versions.props | 8 --- src/installer/pkg/sfx/installers.proj | 4 -- .../pkg/sfx/installers/netstandard2.1.proj | 60 ------------------- 4 files changed, 73 deletions(-) delete mode 100644 src/installer/pkg/sfx/installers/netstandard2.1.proj diff --git a/eng/Subsets.props b/eng/Subsets.props index 00ae8a31c58..a35f99ef759 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -507,7 +507,6 @@ - diff --git a/eng/Versions.props b/eng/Versions.props index 92d5aeedb2d..9ef3967e0ed 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -260,12 +260,4 @@ 8.0.101 $(MicrosoftDotnetSdkInternalVersion) - - - true - diff --git a/src/installer/pkg/sfx/installers.proj b/src/installer/pkg/sfx/installers.proj index 2f947e6d9e2..7f4ce6b9c1c 100644 --- a/src/installer/pkg/sfx/installers.proj +++ b/src/installer/pkg/sfx/installers.proj @@ -22,10 +22,6 @@ - - - - - - true - true - netstandard-targeting-pack-2.1 - NETStandard.Library.Ref - NETStandard.Library.Ref 2.1.0 - false - ToolPack - true - netstandard-targeting-pack-2.1.0-x64.rpm - https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/$(OriginalNETStandard21PkgFilename) - $([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'download')) - $(NETStandard21TempDir)$(OriginalNETStandard21PkgFilename) - $(NETStandard21TempDir)netstandard21.semaphore - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From affe9fc3a8ee9b537ff9d9d63aa863f208ed1ed1 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 15:09:13 -0700 Subject: [PATCH 104/151] Update dependencies from https://github.com/dotnet/emsdk build 20240502.3 (#101817) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.5-servicing.24215.1 -> To Version 8.0.6-servicing.24252.3 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NuGet.config b/NuGet.config index e768edae2ab..58277f76a14 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index be70162daab..c2d1b0ca3f6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 6a06b5454e26309fe4ec786857853056c2c541e0 + a1cd44fdc64aa1f1c4630ddcd95580800d229180 - + https://github.com/dotnet/emsdk - 6a06b5454e26309fe4ec786857853056c2c541e0 + a1cd44fdc64aa1f1c4630ddcd95580800d229180 diff --git a/eng/Versions.props b/eng/Versions.props index 9ef3967e0ed..fcd7ee550f6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -240,7 +240,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.5 + 8.0.6 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From 783771d6acf9b3e656b9392bd53301f999428e67 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 2 May 2024 20:15:35 -0500 Subject: [PATCH 105/151] Pin net7.0 to 7.0.19 (#101823) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index fcd7ee550f6..ba8e464d4f3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,8 +7,8 @@ 0 6 8.0.100 - 7.0.$([MSBuild]::Add($(PatchVersion),14)) - 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) + 7.0.19 + 6.0.$([MSBuild]::Add($(PatchVersion),25)) servicing From 0fdb1335fd9f7fa59552626037c8aa41d06c980b Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Fri, 10 May 2024 11:34:14 -0700 Subject: [PATCH 106/151] [release/8.0-staging] Transition to 1ES templates work (Backport crossgen2 changes) (#101869) * Add check for tizen in ilc and crossgen2 (#90310) * Enable trimming for crossgen2 publishing and fix symbol bug (#91135) * Use a different crossgen2 when running crossgen2 during our build than the crossgen2 that we are shipping (#92677) * Remove explicit listing of RIDs to avoid trying to restore the runtime packs for all possible targets. Use the local targeting and runtime packs explicitly to make sure we're publishing crossgen2 with the local build. Extracted from https://github.com/dotnet/runtime/pull/92826 --------- Co-authored-by: Aleksandr Shaurtaev <38426614+ashaurtaev@users.noreply.github.com> Co-authored-by: Andy Gocke Co-authored-by: Jeremy Koritzinsky Co-authored-by: Jeremy Koritzinsky --- eng/Subsets.props | 2 +- eng/targetingpacks.targets | 6 +- src/coreclr/crossgen-corelib.proj | 3 +- .../tools/aot/ILCompiler/ILCompiler.csproj | 1 + .../tools/aot/crossgen2/crossgen2.csproj | 119 ++---------------- .../tools/aot/crossgen2/crossgen2.props | 1 - .../aot/crossgen2/crossgen2_crossarch.csproj | 8 -- .../aot/crossgen2/crossgen2_inbuild.csproj | 9 ++ .../aot/crossgen2/crossgen2_publish.csproj | 105 ++++++++++++++++ .../Microsoft.NETCore.App.Crossgen2.sfxproj | 42 +++---- .../Microsoft.NETCore.App/ReadyToRun.targets | 4 +- 11 files changed, 147 insertions(+), 153 deletions(-) delete mode 100644 src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj create mode 100644 src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj create mode 100644 src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj diff --git a/eng/Subsets.props b/eng/Subsets.props index a35f99ef759..69924723836 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -342,7 +342,7 @@ - + diff --git a/eng/targetingpacks.targets b/eng/targetingpacks.targets index b3ad8560d63..e9ca4e11615 100644 --- a/eng/targetingpacks.targets +++ b/eng/targetingpacks.targets @@ -35,7 +35,7 @@ LatestRuntimeFrameworkVersion="$(ProductVersion)" RuntimeFrameworkName="$(LocalFrameworkOverrideName)" RuntimePackNamePatterns="$(LocalFrameworkOverrideName).Runtime.**RID**" - RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86" + RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;freebsd-x64;freebsd-arm64" TargetFramework="$(NetCoreAppCurrent)" TargetingPackName="$(LocalFrameworkOverrideName).Ref" TargetingPackVersion="$(ProductVersion)" @@ -104,6 +104,10 @@ Condition="'$(UsePackageDownload)' == 'true' and $([System.String]::Copy('%(Identity)').StartsWith('$(LocalFrameworkOverrideName).Runtime'))" /> + + - - + diff --git a/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj b/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj index 7c25444f209..c0a239225fa 100644 --- a/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj +++ b/src/coreclr/tools/aot/ILCompiler/ILCompiler.csproj @@ -12,6 +12,7 @@ $(RuntimeBinDir)ilc-published/ false + false false true diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj index ef1b82f537f..6f8578b26b9 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.csproj +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.csproj @@ -1,113 +1,14 @@ - - - - + - $(RuntimeBinDir)crossgen2 - - false - - true - linux-x64;linux-musl-x64;linux-arm;linux-musl-arm;linux-arm64;linux-musl-arm64;freebsd-x64;freebsd-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 - $(PackageRID) - false - true + $(RuntimeBinDir)/crossgen2 + false + + $(NetCoreAppToolCurrent) - - - - true - true - - false - - false - true - - - - - - $(CoreCLRILCompilerDir) - $(CoreCLRCrossILCompilerDir) - $(ROOTFS_DIR) - $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll - $(CoreCLRAotSdkDir) - $(MicrosoftNetCoreAppRuntimePackRidLibTfmDir) - $(MicrosoftNetCoreAppRuntimePackNativeDir) - false - - .dwarf - --flat - - - - - - - - - - - - - - - - - $(MicrosoftNetCoreAppRuntimePackDir) - - - - - - - - - $(RuntimeIdentifier) - - - x86_64 - aarch64 - arm64 - - - $(CrossCompileArch)-linux-gnu - $(CrossCompileArch)-alpine-linux-musl - $(CrossCompileArch)-unknown-freebsd12 - - - - - - - - - - clang - - - - - - - - $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) - <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) - lld - - - diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2.props b/src/coreclr/tools/aot/crossgen2/crossgen2.props index 0f2f954c6a1..4d5cc30c3bb 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2.props +++ b/src/coreclr/tools/aot/crossgen2/crossgen2.props @@ -3,7 +3,6 @@ crossgen2 true Exe - $(NetCoreAppToolCurrent) 8002,NU1701 x64;x86;arm64;arm;loongarch64 AnyCPU diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj deleted file mode 100644 index 544b4271117..00000000000 --- a/src/coreclr/tools/aot/crossgen2/crossgen2_crossarch.csproj +++ /dev/null @@ -1,8 +0,0 @@ - - - $(BuildArchitecture) - $(RuntimeBinDir)/$(CrossHostArch)/crossgen2 - false - - - diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj new file mode 100644 index 00000000000..75766fd75ea --- /dev/null +++ b/src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj @@ -0,0 +1,9 @@ + + + $(BuildArchitecture) + $(RuntimeBinDir)/$(BuildArchitecture)/crossgen2 + false + $(NetCoreAppToolCurrent) + + + diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj new file mode 100644 index 00000000000..d5105eccff3 --- /dev/null +++ b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj @@ -0,0 +1,105 @@ + + + + + + false + false + true + $(PackageRID) + true + $(NetCoreAppCurrent) + true + true + + + + + + true + true + + false + + false + true + + + + + + + + + + + + + $(CoreCLRILCompilerDir) + $(CoreCLRCrossILCompilerDir) + $(ROOTFS_DIR) + $(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll + $(CoreCLRAotSdkDir) + $(MicrosoftNetCoreAppRuntimePackRidLibTfmDir) + $(MicrosoftNetCoreAppRuntimePackNativeDir) + false + + .dwarf + --flat + + + + + + + + + + + + + + + $(RuntimeIdentifier) + + + x86_64 + aarch64 + arm64 + + + $(CrossCompileArch)-linux-gnu + $(CrossCompileArch)-alpine-linux-musl + $(CrossCompileArch)-unknown-freebsd12 + + + + + + + + + + clang + + + + + + + + $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) + <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) + lld + + + + diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj index e8e8591bbaf..a95eb81b931 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj @@ -28,36 +28,22 @@ + + + + - - - - - - - - - - <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')" KeepMetadata="REMOVE_ALL" /> diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets index 0b82891e6c4..6b24c54f4c1 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/ReadyToRun.targets @@ -14,9 +14,7 @@ - - $(BuildArchitecture) - $(CoreCLRArtifactsPath)\$(CrossDir)\crossgen2\crossgen2.dll + $(CoreCLRArtifactsPath)\$(BuildArchitecture)\crossgen2\crossgen2.dll true @(PublishReadyToRunCrossgen2ExtraArgsList) From 76b5685d7ec25a12d1b1abd6a712308bc7699bca Mon Sep 17 00:00:00 2001 From: Radek Zikmund <32671551+rzikm@users.noreply.github.com> Date: Mon, 13 May 2024 22:29:58 +0200 Subject: [PATCH 107/151] [release/8.0] Fix data race leading to a deadlock when opening QuicStream (#102147) * Fix data race leading to a deadlock when opening QuicStream (#101250) * Fix data race leading to a deadlock. * Remove unwanted change * Code review feedback * Fix hang * Add assert * Fix potential crash * Code review feedback * Fix thrown exception. --- .../System/Net/Quic/Internal/ThrowHelper.cs | 27 +++++++++++++++---- .../src/System/Net/Quic/QuicConnection.cs | 23 +++++++++++++--- .../src/System/Net/Quic/QuicStream.cs | 14 +++++++--- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs index ec677f9f4e5..e99f5fda640 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/ThrowHelper.cs @@ -27,13 +27,27 @@ internal static QuicException GetOperationAbortedException(string? message = nul return new QuicException(QuicError.OperationAborted, null, message ?? SR.net_quic_operationaborted); } - internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWhen(true)] out Exception? exception) + internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWhen(true)] out Exception? exception, bool streamWasSuccessfullyStarted = true, string? message = null) { if (status == QUIC_STATUS_ABORTED) { - // If status == QUIC_STATUS_ABORTED, we will receive an event later, which will complete the task source. - exception = null; - return false; + // Connection has been closed by the peer (either at transport or application level), + if (streamWasSuccessfullyStarted) + { + // we will receive an event later, which will complete the stream with concrete + // information why the connection was aborted. + exception = null; + return false; + } + else + { + // we won't be receiving any event callback for shutdown on this stream, so we don't + // necessarily know which error to report. So we throw an exception which we can distinguish + // at the caller (ConnectionAborted normally has App error code) and throw the correct + // exception from there. + exception = new QuicException(QuicError.ConnectionAborted, null, ""); + return true; + } } else if (status == QUIC_STATUS_INVALID_STATE) { @@ -43,13 +57,16 @@ internal static bool TryGetStreamExceptionForMsQuicStatus(int status, [NotNullWh } else if (StatusFailed(status)) { - exception = GetExceptionForMsQuicStatus(status); + exception = GetExceptionForMsQuicStatus(status, message: message); return true; } exception = null; return false; } + // see TryGetStreamExceptionForMsQuicStatus for explanation + internal static bool IsConnectionAbortedWhenStartingStreamException(Exception ex) => ex is QuicException qe && qe.QuicError == QuicError.ConnectionAborted && qe.ApplicationErrorCode is null; + internal static Exception GetExceptionForMsQuicStatus(int status, long? errorCode = default, string? message = null) { Exception ex = GetExceptionInternal(status, errorCode, message); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs index a2ade033afe..13351faaa20 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs @@ -98,6 +98,11 @@ static async ValueTask StartConnectAsync(QuicClientConnectionOpt /// private int _disposed; + /// + /// Completed when connection shutdown is initiated. + /// + private TaskCompletionSource _connectionCloseTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly ValueTaskSource _connectedTcs = new ValueTaskSource(); private readonly ValueTaskSource _shutdownTcs = new ValueTaskSource(); @@ -376,16 +381,22 @@ public async ValueTask OpenOutboundStreamAsync(QuicStreamType type, stream = new QuicStream(_handle, type, _defaultStreamErrorCode); await stream.StartAsync(cancellationToken).ConfigureAwait(false); } - catch + catch (Exception ex) { if (stream is not null) { await stream.DisposeAsync().ConfigureAwait(false); } + + // In case of an incoming race when the connection is closed by the peer just before we open the stream, + // we receive QUIC_STATUS_ABORTED from MsQuic, but we don't know how the connection was closed. We throw + // special exception and handle it here where we can determine the shutdown reason. + bool connectionAbortedByPeer = ThrowHelper.IsConnectionAbortedWhenStartingStreamException(ex); + // Propagate connection error if present. - if (_acceptQueue.Reader.Completion.IsFaulted) + if (_connectionCloseTcs.Task.IsFaulted || connectionAbortedByPeer) { - await _acceptQueue.Reader.Completion.ConfigureAwait(false); + await _connectionCloseTcs.Task.ConfigureAwait(false); } throw; } @@ -475,17 +486,21 @@ private unsafe int HandleEventShutdownInitiatedByTransport(ref SHUTDOWN_INITIATE { Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetExceptionForMsQuicStatus(data.Status, (long)data.ErrorCode)); _connectedTcs.TrySetException(exception); + _connectionCloseTcs.TrySetException(exception); _acceptQueue.Writer.TryComplete(exception); return QUIC_STATUS_SUCCESS; } private unsafe int HandleEventShutdownInitiatedByPeer(ref SHUTDOWN_INITIATED_BY_PEER_DATA data) { - _acceptQueue.Writer.TryComplete(ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetConnectionAbortedException((long)data.ErrorCode))); + Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetConnectionAbortedException((long)data.ErrorCode)); + _connectionCloseTcs.TrySetException(exception); + _acceptQueue.Writer.TryComplete(exception); return QUIC_STATUS_SUCCESS; } private unsafe int HandleEventShutdownComplete() { Exception exception = ExceptionDispatchInfo.SetCurrentStackTrace(ThrowHelper.GetOperationAbortedException()); + _connectionCloseTcs.TrySetException(exception); _acceptQueue.Writer.TryComplete(exception); _connectedTcs.TrySetException(exception); _shutdownTcs.TrySetResult(); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs index 6165f2085cb..82ee656dc6b 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicStream.cs @@ -161,13 +161,18 @@ internal unsafe QuicStream(MsQuicContextSafeHandle connectionHandle, QuicStreamT try { QUIC_HANDLE* handle; - ThrowHelper.ThrowIfMsQuicError(MsQuicApi.Api.StreamOpen( + int status = MsQuicApi.Api.StreamOpen( connectionHandle, type == QuicStreamType.Unidirectional ? QUIC_STREAM_OPEN_FLAGS.UNIDIRECTIONAL : QUIC_STREAM_OPEN_FLAGS.NONE, &NativeCallback, (void*)GCHandle.ToIntPtr(context), - &handle), - "StreamOpen failed"); + &handle); + + if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? ex, streamWasSuccessfullyStarted: false, message: "StreamOpen failed")) + { + throw ex; + } + _handle = new MsQuicContextSafeHandle(handle, context, SafeHandleType.Stream, connectionHandle); } catch @@ -241,7 +246,8 @@ internal ValueTask StartAsync(CancellationToken cancellationToken = default) int status = MsQuicApi.Api.StreamStart( _handle, QUIC_STREAM_START_FLAGS.SHUTDOWN_ON_FAIL | QUIC_STREAM_START_FLAGS.INDICATE_PEER_ACCEPT); - if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? exception)) + + if (ThrowHelper.TryGetStreamExceptionForMsQuicStatus(status, out Exception? exception, streamWasSuccessfullyStarted: false)) { _startedTcs.TrySetException(exception); } From bb432d7f3ade0f79671717a5c10ef55d7d82a9a1 Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Tue, 14 May 2024 12:58:24 -0700 Subject: [PATCH 108/151] [release/8.0-staging] Backport "Merge CoreCLR-based official builds legs into one leg per platform" (#102097) * Merge CoreCLR-based official builds legs into one leg per platform (#92901) * Pass "is official build" variable to AllConfigurations leg in the official build. (#93078) --------- Co-authored-by: Jeremy Koritzinsky --- eng/pipelines/common/global-build-job.yml | 9 +- .../common/templates/global-build-step.yml | 12 + .../common/templates/runtimes/xplat-job.yml | 9 - eng/pipelines/common/xplat-setup.yml | 9 + eng/pipelines/coreclr/templates/build-job.yml | 21 -- .../coreclr/templates/crossdac-build.yml | 80 ---- .../coreclr/templates/crossdac-hostarch.yml | 9 + .../coreclr/templates/crossdac-pack.yml | 75 ---- .../coreclr/templates/xplat-pipeline-job.yml | 6 - eng/pipelines/installer/jobs/build-job.yml | 29 +- .../jobs/steps/build-linux-package.yml | 2 + eng/pipelines/runtime-official.yml | 344 +++++++++++++----- eng/pipelines/runtime.yml | 23 +- src/coreclr/build-runtime.cmd | 9 +- 14 files changed, 317 insertions(+), 320 deletions(-) create mode 100644 eng/pipelines/common/templates/global-build-step.yml delete mode 100644 eng/pipelines/coreclr/templates/crossdac-build.yml create mode 100644 eng/pipelines/coreclr/templates/crossdac-hostarch.yml delete mode 100644 eng/pipelines/coreclr/templates/crossdac-pack.yml diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index cdda8c99dc5..4b06818b140 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -237,10 +237,11 @@ jobs: - task: CodeQL3000Init@0 displayName: Initialize CodeQL (manually-injected) - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci $(_archParameter) $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) - displayName: Build product - ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: - continueOnError: ${{ parameters.shouldContinueOnError }} + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: ${{ parameters.buildArgs }} + useContinueOnErrorDuringBuild: ${{ parameters.useContinueOnErrorDuringBuild }} + shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - ${{ if eq(parameters.isManualCodeQLBuild, true) }}: - task: CodeQL3000Finalize@0 diff --git a/eng/pipelines/common/templates/global-build-step.yml b/eng/pipelines/common/templates/global-build-step.yml new file mode 100644 index 00000000000..7f38a9fd118 --- /dev/null +++ b/eng/pipelines/common/templates/global-build-step.yml @@ -0,0 +1,12 @@ +parameters: + buildArgs: '' + useContinueOnErrorDuringBuild: false + shouldContinueOnError: false + archParameter: $(_archParameter) + displayName: Build product + +steps: + - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci ${{ parameters.archParameter }} $(_osParameter) $(crossArg) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_buildDarwinFrameworksParameter) $(_overrideTestScriptWindowsCmdParameter) + displayName: ${{ parameters.displayName }} + ${{ if eq(parameters.useContinueOnErrorDuringBuild, true) }}: + continueOnError: ${{ parameters.shouldContinueOnError }} diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index f4ac7e82957..23e74c70e57 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -77,15 +77,6 @@ jobs: - name: buildConfig value: ${{ parameters.buildConfig }} - - name: archType - value: ${{ parameters.archType }} - - - name: osGroup - value: ${{ parameters.osGroup }} - - - name: osSubgroup - value: ${{ parameters.osSubgroup }} - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - name: _HelixSource value: official/dotnet/runtime/$(Build.SourceBranch) diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index eb19570aeec..675a2679201 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -45,6 +45,15 @@ jobs: - name: _BuildConfig value: $(buildConfigUpper) + + - name: archType + value: ${{ parameters.archType }} + + - name: osGroup + value: ${{ parameters.osGroup }} + + - name: osSubgroup + value: ${{ parameters.osSubgroup }} - name: _runSmokeTestsOnlyArg value: '/p:RunSmokeTestsOnly=$(isRunSmokeTestsOnly)' diff --git a/eng/pipelines/coreclr/templates/build-job.yml b/eng/pipelines/coreclr/templates/build-job.yml index e995e626a56..ebaeae5e124 100644 --- a/eng/pipelines/coreclr/templates/build-job.yml +++ b/eng/pipelines/coreclr/templates/build-job.yml @@ -231,27 +231,6 @@ jobs: artifactName: $(buildProductArtifactName) displayName: 'product build' - - ${{ if and(in(parameters.osGroup, 'windows', 'linux'), ne(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: ${{ parameters.archType }} - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - ${{ if eq(parameters.archType, 'arm') }}: - hostArchType: x86 - ${{ else }}: - hostArchType: x64 - - - ${{ if and(in(parameters.osGroup, 'windows'), eq(parameters.archType, 'x86')) }}: - - template: /eng/pipelines/coreclr/templates/crossdac-build.yml - parameters: - archType: arm - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - isOfficialBuild: ${{ parameters.signBinaries }} - hostArchType: x86 - - ${{ if and(ne(parameters.testGroup, ''), ne(parameters.disableClrTest, true)) }}: # Publish test native components for consumption by test execution. - ${{ if ne(parameters.isOfficialBuild, true) }}: diff --git a/eng/pipelines/coreclr/templates/crossdac-build.yml b/eng/pipelines/coreclr/templates/crossdac-build.yml deleted file mode 100644 index a8accb9d2e3..00000000000 --- a/eng/pipelines/coreclr/templates/crossdac-build.yml +++ /dev/null @@ -1,80 +0,0 @@ -parameters: - archType: '' - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - hostArchType: '' - -steps: - # Always build the crossdac, that way we know in CI/PR if things break to build. - - ${{ if and(eq(parameters.osGroup, 'windows'), notin(parameters.archType, 'x86')) }}: - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os linux -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux DAC for Windows - - - script: $(Build.SourcesDirectory)/src/coreclr/build-runtime$(scriptExt) $(buildConfig) ${{ parameters.archType }} -hostarch ${{ parameters.hostArchType }} -ci -os alpine -cmakeargs "-DCLR_CROSS_COMPONENTS_BUILD=1" -ninja $(officialBuildIdArg) -component crosscomponents - displayName: Build Cross OS Linux-musl DAC for Windows - - - powershell: | - function CopyAndVerifyCrossOsAssets { - [CmdletBinding()] - param ( - [Parameter(Mandatory)][string]$crossDacDir, - [Parameter(Mandatory)][string]$targetDir - ) - - $crossDacDir = Join-Path $crossDacDir -ChildPath '${{ parameters.hostArchType }}' - - $availableFiles = ls -File $crossDacDir - - Write-Host "Probed for files in ""$crossDacDir"", found:" - $availableFiles | fl - - if (-not ("mscordaccore.dll" -in $availableFiles.Name ` - -and "mscordaccore.pdb" -in $availableFiles.Name ` - -and "mscordbi.dll" -in $availableFiles.Name ` - -and "mscordbi.pdb" -in $availableFiles.Name` - )) - { - Write-Error "Couldn't find one of the expected crossdac files." - } - - New-Item $targetDir -ItemType 'Directory' -Force -ea 0 - $availableFiles | %{ cp $_.FullName $targetDir -v } - } - - $buildMuslDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/alpine.${{ parameters.archType }}.$(buildConfigUpper)" - $buildMuslStagingPath = "$(crossDacArtifactPath)/Linux_musl.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - $buildLinuxDacRootFolderPath = "$(Build.SourcesDirectory)/artifacts/bin/coreclr/Linux.${{ parameters.archType }}.$(buildConfigUpper)" - $buildLinuxDacStagingPath = "$(crossDacArtifactPath)/Linux.${{ parameters.archType }}.$(buildConfigUpper)/${{ parameters.hostArchType }}" - - - CopyAndVerifyCrossOsAssets -CrossDacDir $buildMuslDacRootFolderPath -TargetDir $buildMuslStagingPath - CopyAndVerifyCrossOsAssets -CrossDacDir $buildLinuxDacRootFolderPath -TargetDir $buildLinuxDacStagingPath - - Write-Host "Final directory contents:" - ls -R $(crossDacArtifactPath) - - displayName: Gather CrossDac Artifacts - - - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml - parameters: - basePath: $(crossDacArtifactPath) - isOfficialBuild: ${{ parameters.isOfficialBuild }} - timeoutInMinutes: 30 - - - ${{ if eq(parameters.osGroup, 'linux') }}: - - task: CopyFiles@2 - displayName: Gather runtime for CrossDac - inputs: - SourceFolder: $(coreClrProductRootFolderPath) - Contents: libcoreclr.so - TargetFolder: '$(crossDacArtifactPath)/${{ parameters.osGroup }}${{ parameters.osSubgroup }}.$(archType).$(buildConfigUpper)/${{ parameters.hostArchType }}' - - # Make the assets available in a single container for the packaging job. - - task: PublishBuildArtifacts@1 - displayName: Publish runtime for CrossDac - inputs: - pathtoPublish: $(crossDacArtifactPath) - PublishLocation: Container - artifactName: $(buildCrossDacArtifactName) diff --git a/eng/pipelines/coreclr/templates/crossdac-hostarch.yml b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml new file mode 100644 index 00000000000..e100217a481 --- /dev/null +++ b/eng/pipelines/coreclr/templates/crossdac-hostarch.yml @@ -0,0 +1,9 @@ +parameters: + archType: '' + +variables: + - name: crossDacHostArch + value: x64 + - ${{ if eq(parameters.archType, 'arm') }}: + - name: crossDacHostArch + value: x86 \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/crossdac-pack.yml b/eng/pipelines/coreclr/templates/crossdac-pack.yml deleted file mode 100644 index 520eb65fb33..00000000000 --- a/eng/pipelines/coreclr/templates/crossdac-pack.yml +++ /dev/null @@ -1,75 +0,0 @@ -parameters: - archType: '' - buildConfig: '' - container: '' - crossDacPlatforms: {} - dependOnEvaluatePaths: false - isOfficialBuild: false - osGroup: '' - osSubgroup: '' - platform: '' - pool: '' - runtimeVariant: '' - testGroup: '' - timeoutInMinutes: '' - variables: {} - -jobs: -- template: xplat-pipeline-job.yml - parameters: - archType: ${{ parameters.archType }} - buildConfig: ${{ parameters.buildConfig }} - container: ${{ parameters.container }} - condition: ${{ parameters.isOfficialBuild }} - helixType: 'build/product/' - osGroup: ${{ parameters.osGroup }} - osSubgroup: ${{ parameters.osSubgroup }} - pool: ${{ parameters.pool }} - runtimeVariant: ${{ parameters.runtimeVariant }} - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} - - name: crossdacpack - displayName: CrossDac Packaging - - variables: - - name: officialBuildIdArg - value: '' - - name: crossDacArgs - value: '/p:CrossDacArtifactsDir=$(crossDacArtifactPath)/$(buildCrossDacArtifactName)' - - ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: - - name: officialBuildIdArg - value: '/p:OfficialBuildId=$(Build.BuildNumber)' - - name: SignType - value: $[ coalesce(variables.OfficialSignType, 'real') ] - - ${{ parameters.variables }} - - dependsOn: - - ${{ if ne(parameters.crossDacPlatforms, '') }}: - - ${{ each platform in parameters.crossDacPlatforms }}: - - ${{ parameters.runtimeFlavor }}_${{ parameters.runtimeVariant }}_product_build_${{ platform }}_${{ parameters.buildConfig }} - - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download CrossDac artifacts - inputs: - artifactName: $(buildCrossDacArtifactName) - downloadPath: $(crossDacArtifactPath) - checkDownloadedFiles: true - - - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -subset crossdacpack -arch $(archType) $(osArg) -c $(buildConfig) $(officialBuildIdArg) $(crossDacArgs) -ci - displayName: Build crossdac packaging - - # Save packages using the prepare-signed-artifacts format. - - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - parameters: - name: ${{ parameters.platform }} - - # Upload to artifacts to be signed - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'CrossDacPackagingLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index 162f5e9f0ed..b6c57be0c7f 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -69,12 +69,6 @@ jobs: - name: buildProductRootFolderPath value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)' - - name: buildCrossDacArtifactName - value: CoreCLRCrossDacArtifacts - - - name: crossDacArtifactPath - value: $(Build.SourcesDirectory)/artifacts/$(buildCrossDacArtifactName) - # We need this because both mono and coreclr build currently depends on CoreClr - name: coreClrProductArtifactName value: 'CoreCLRProduct_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' diff --git a/eng/pipelines/installer/jobs/build-job.yml b/eng/pipelines/installer/jobs/build-job.yml index 1c45b437c80..43f19c69cca 100644 --- a/eng/pipelines/installer/jobs/build-job.yml +++ b/eng/pipelines/installer/jobs/build-job.yml @@ -339,32 +339,9 @@ jobs: df -h displayName: Disk Usage before Build - # Build the default subset non-MacOS platforms - - ${{ if ne(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) - displayName: Build - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - # Build corehost, sign and add entitlements to MacOS binaries - - ${{ if eq(parameters.osGroup, 'osx') }}: - - script: $(BaseJobBuildCommand) -subset host.native - displayName: Build CoreHost - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: /eng/pipelines/common/macos-sign-with-entitlements.yml - parameters: - filesToSign: - - name: dotnet - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - name: apphost - path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost - entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist - - - script: $(BaseJobBuildCommand) -subset host.pkg+host.tools+host.tests+packs - displayName: Build and Package - continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} + - script: $(BaseJobBuildCommand) + displayName: Build + continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }} - ${{ if and(eq(parameters.isOfficialBuild, true), eq(parameters.osGroup, 'windows')) }}: - powershell: ./eng/collect_vsinfo.ps1 -ArchiveRunName postbuild_log diff --git a/eng/pipelines/installer/jobs/steps/build-linux-package.yml b/eng/pipelines/installer/jobs/steps/build-linux-package.yml index a5645af3d9c..102eab770c2 100644 --- a/eng/pipelines/installer/jobs/steps/build-linux-package.yml +++ b/eng/pipelines/installer/jobs/steps/build-linux-package.yml @@ -4,6 +4,7 @@ parameters: packageStepDescription: null packagingArgs: '' subsetArg: '' + condition: succeeded() steps: ## Run NuGet Authentication for each of the side containers @@ -20,6 +21,7 @@ steps: /bl:artifacts/log/$(_BuildConfig)/msbuild.${{ parameters.packageType }}.installers.binlog displayName: Package ${{ parameters.packageStepDescription }} - ${{ parameters.packageType }} target: ${{ parameters.target }} + condition: ${{ parameters.condition }} # Broken symbolic links break the SBOM processing # We make some symlinks during the installer generation process, # but they aren't always valid on disk afterwards. Some of our tooling, diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 3138f18ebd7..6e6781bb134 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -58,47 +58,246 @@ extends: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true # - # Build CoreCLR + # Build CoreCLR runtime packs + # Windows x64/arm64 + # Sign diagnostic files after native build # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + - windows_arm64 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + + # + # Build CoreCLR runtime packs + # Windows x86 + # No NativeAOT as NativeAOT is not supported on x86 + # Sign diagnostic files after native build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x86 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(Build.SourcesDirectory)/artifacts/bin/coreclr + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + # Now that we've signed the diagnostic files, do the rest of the build. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.tools+clr.packages+libs+host+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR components, all libraries, hosts, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + # + # Build CoreCLR runtime packs + # Mac x64/arm64 + # Sign and entitle createdump and corerun after native build. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - osx_arm64 - osx_x64 + jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog + nameSuffix: CoreCLR + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 120 + postBuildSteps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: /eng/pipelines/common/macos-sign-with-entitlements.yml + parameters: + filesToSign: + - name: createdump + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/createdump-entitlements.plist + - name: corerun + path: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: dotnet + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + - name: apphost + path: $(Build.SourcesDirectory)/artifacts/bin/$(osGroup)-$(archType).$(_BuildConfig)/corehost + entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist + + - task: CopyFiles@2 + displayName: 'Copy signed createdump to sharedFramework' + inputs: + contents: createdump + sourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + targetFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)/sharedFramework + overWrite: true + + # Now that we've entitled and signed createdump, we can build the rest. + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.corelib+clr.nativecorelib+clr.nativeaotlibs+clr.tools+clr.packages+libs+host.tools+host.pkg+packs -c $(_BuildConfig) + displayName: Build managed CoreCLR and host components, all libraries, and packs + + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + + # + # Build CoreCLR runtime packs + # Linux and Linux_musl + # CoreCLR runtime for CrossDac packaging + # Create Linux installers + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: - linux_x64 - linux_arm - linux_arm64 - linux_musl_x64 - linux_musl_arm - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 jobParameters: + buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.aot+clr.packages+libs+host+packs -c $(_BuildConfig) + nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} - signBinaries: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 + postBuildSteps: + # Upload libcoreclr.so for CrossDac packaging + - task: CopyFiles@2 + displayName: Gather runtime for CrossDac + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) + Contents: libcoreclr.so + TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch) + - task: PublishBuildArtifacts@1 + displayName: Publish runtime for CrossDac + inputs: + pathToPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts + PublishLocation: Container + artifactName: CoreCLRCrossDacArtifacts + # Create RPMs and DEBs + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: deb + target: debpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs Deb installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildDebPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), eq(variables.archType, 'x64')) + - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml + parameters: + packageType: rpm + target: rpmpkg + packageStepDescription: Runtime Deps, Runtime, Framework Packs RPM installers + subsetArg: -s packs.installers + packagingArgs: -c $(_BuildConfig) --arch $(archType) --os $(osGroup) --ci /p:OfficialBuildId=$(Build.BuildNumber) /p:BuildRpmPackage=true + condition: and(succeeded(), eq(variables.osSubgroup, ''), in(variables.archType, 'x64', 'arm64')) + # Upload the results. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: $(osGroup)$(osSubgroup)_$(archType) + extraVariablesTemplates: + - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml + + # + # Build and Pack CrossDac + # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/coreclr/templates/crossdac-pack.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: release platforms: - windows_x64 jobParameters: + buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) + nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 - crossDacPlatforms: - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x64 - - windows_arm64 + preBuildSteps: + - task: DownloadBuildArtifacts@0 + displayName: Download Runtimes for CrossDac packaging + inputs: + artifactName: $(crossDacArtifactsContainer) + downloadPath: $(crossDacArtifactsBasePath) + checkDownloadedFiles: true + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s linuxdac+alpinedac -c $(_BuildConfig) + archParameter: -arch x64,x86,arm,arm64 + - task: CopyFiles@2 + displayName: Gather CrossDacs + inputs: + SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr + Contents: | + ** + !**\sharedFramework\** + TargetFolder: $(crossDacArtifactsPath) + - template: /eng/pipelines/coreclr/templates/sign-diagnostic-files.yml + parameters: + basePath: $(crossDacArtifactsPath) + isOfficialBuild: ${{ variables.isOfficialBuild }} + timeoutInMinutes: 30 + postBuildSteps: + # Save packages using the prepare-signed-artifacts format. + # CrossDac packages are expected to be in the windows_x64 folder. + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: windows_x64 + dependsOn: + - build_linux_x64_release_CoreCLR + - build_linux_arm_release_CoreCLR + - build_linux_arm64_release_CoreCLR + - build_linux_musl_x64_release_CoreCLR + - build_linux_musl_arm_release_CoreCLR + - build_linux_musl_arm64_release_CoreCLR + variables: + - name: crossDacArtifactsContainer + value: CoreCLRCrossDacArtifacts + - name: crossDacArtifactsBasePath + value: $(Build.StagingDirectory)/CrossDac + - name: crossDacArtifactsPath + value: $(crossDacArtifactsBasePath)/$(crossDacArtifactsContainer) # # Build NativeAOT runtime packs @@ -122,7 +321,7 @@ extends: - linux_bionic_x64 jobParameters: buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true - nameSuffix: AllSubsets_NativeAOT + nameSuffix: NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -165,7 +364,7 @@ extends: # - windows_arm64 jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -182,7 +381,7 @@ extends: - wasi_wasm jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono + nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -198,7 +397,7 @@ extends: - browser_wasm jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoWasmBuildVariant=multithread /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - nameSuffix: AllSubsets_Mono_multithread + nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} runtimeVariant: multithread postBuildSteps: @@ -329,7 +528,7 @@ extends: jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false - nameSuffix: AllSubsets_Mono_LLVMJIT + nameSuffix: Mono_LLVMJIT runtimeVariant: LLVMJIT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -343,7 +542,7 @@ extends: jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - nameSuffix: AllSubsets_Mono_LLVMAOT + nameSuffix: Mono_LLVMAOT runtimeVariant: LLVMAOT isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: @@ -352,49 +551,26 @@ extends: name: MonoRuntimePacks # - # Build libraries using live CoreLib from CoreCLR + # Build libraries AllConfigurations for packages # - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml + jobTemplate: /eng/pipelines/common/global-build-job.yml buildConfig: Release platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - windows_x64 - - windows_arm64 jobParameters: + buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true + nameSuffix: Libraries_AllConfigurations isOfficialBuild: ${{ variables.isOfficialBuild }} - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: Libraries_AllConfigurations + timeoutInMinutes: 95 # - # Build libraries AllConfigurations for packages + # Build SourceBuild packages # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/libraries/build-job.yml - buildConfig: Release - platforms: - - windows_x64 - jobParameters: - framework: allConfigurations - isOfficialBuild: ${{ variables.isOfficialBuild }} - isOfficialAllConfigurations: true - liveRuntimeBuildConfig: release - # Official builds don't run tests, locally or on Helix - runTests: false - useHelix: false - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml @@ -410,30 +586,6 @@ extends: name: SourceBuildPackages timeoutInMinutes: 95 - # - # Installer Build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/installer/jobs/build-job.yml - buildConfig: Release - jobParameters: - liveRuntimeBuildConfig: release - liveLibrariesBuildConfig: Release - isOfficialBuild: ${{ variables.isOfficialBuild }} - platforms: - - osx_arm64 - - osx_x64 - - linux_x64 - - linux_arm - - linux_arm64 - - linux_musl_x64 - - linux_musl_arm - - linux_musl_arm64 - - windows_x86 - - windows_x64 - - windows_arm64 - # # Build PGO Instrumented CoreCLR Release # @@ -470,24 +622,24 @@ extends: isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 dependsOn: - - Build_android_arm_release_AllSubsets_Mono - - Build_android_arm64_release_AllSubsets_Mono - - Build_android_x86_release_AllSubsets_Mono - - Build_android_x64_release_AllSubsets_Mono - - Build_browser_wasm_Linux_release_AllSubsets_Mono - - Build_wasi_wasm_linux_release_AllSubsets_Mono - - Build_ios_arm64_release_AllSubsets_Mono - - Build_iossimulator_x64_release_AllSubsets_Mono - - Build_iossimulator_arm64_release_AllSubsets_Mono - - Build_maccatalyst_arm64_release_AllSubsets_Mono - - Build_maccatalyst_x64_release_AllSubsets_Mono - - Build_tvos_arm64_release_AllSubsets_Mono - - Build_tvossimulator_arm64_release_AllSubsets_Mono - - Build_tvossimulator_x64_release_AllSubsets_Mono + - Build_android_arm_release_Mono + - Build_android_arm64_release_Mono + - Build_android_x86_release_Mono + - Build_android_x64_release_Mono + - Build_browser_wasm_Linux_release_Mono + - Build_wasi_wasm_linux_release_Mono + - Build_ios_arm64_release_Mono + - Build_iossimulator_x64_release_Mono + - Build_iossimulator_arm64_release_Mono + - Build_maccatalyst_arm64_release_Mono + - Build_maccatalyst_x64_release_Mono + - Build_tvos_arm64_release_Mono + - Build_tvossimulator_arm64_release_Mono + - Build_tvossimulator_x64_release_Mono - Build_windows_x64_release_CrossAOT_Mono - - installer_coreclr__windows_x64_Release_ - - installer_coreclr__windows_x86_Release_ - - installer_coreclr__windows_arm64_Release_ + - Build_windows_x64_release_CoreCLR + - Build_windows_x86_release_CoreCLR + - Build_windows_arm64_release_CoreCLR - ${{ if eq(variables.isOfficialBuild, true) }}: - template: /eng/pipelines/official/stages/publish.yml diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index c83a76993e6..767b3dc9ba4 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -400,6 +400,27 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_tools_illink.containsChange'], true), eq(variables['isRollingBuild'], true)) + # + # Build CrossDacs + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + platforms: + - windows_x64 + variables: + - name: _archParameter + value: -arch x64,x86,arm,arm64 + jobParameters: + buildArgs: -s linuxdac+alpinedac -c Checked,$(_BuildConfig) + nameSuffix: CrossDac + isOfficialBuild: false + timeoutInMinutes: 60 + postBuildSteps: + - publish: $(Build.SourcesDirectory)/artifacts/bin/coreclr + displayName: Publish CrossDacs for diagnostics + artifact: CoreCLRCrossDacArtifacts # Build Mono AOT offset headers once, for consumption elsewhere # Only when mono changed @@ -415,7 +436,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: - isOfficialBuild: ${{ variables.isOfficialBuild }} + isOfficialBuild: false # needed by crossaot condition: >- or( diff --git a/src/coreclr/build-runtime.cmd b/src/coreclr/build-runtime.cmd index ec6887c78d8..d85e6c95909 100644 --- a/src/coreclr/build-runtime.cmd +++ b/src/coreclr/build-runtime.cmd @@ -211,8 +211,13 @@ if NOT "%__BuildType%"=="Release" ( set __PgoOptimize=0 ) -set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOS%.%__TargetArch%.%__BuildType%" -set "__IntermediatesDir=%__RootBinDir%\obj\coreclr\%__TargetOS%.%__TargetArch%.%__BuildType%" +set __TargetOSDirName=%__TargetOS% +if "%__TargetOS%"=="alpine" ( + set __TargetOSDirName=linux_musl +) + +set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOSDirName%.%__TargetArch%.%__BuildType%" +set "__IntermediatesDir=%__RootBinDir%\obj\coreclr\%__TargetOSDirName%.%__TargetArch%.%__BuildType%" set "__LogsDir=%__RootBinDir%\log\!__BuildType!" set "__MsbuildDebugLogsDir=%__LogsDir%\MsbuildDebugLogs" set "__ArtifactsIntermediatesDir=%__RepoRootDir%\artifacts\obj\coreclr\" From 98dd7f85c85fb89ccb82ce24d3eb630fce3057e3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 23:24:56 +0200 Subject: [PATCH 109/151] [release/8.0-staging] [WinHttpHandler] Move `_cachedSendPinnedBuffer` ownership to `WinHttpRequestState` (#102083) * move _cachedSendPinnedBuffer ownership to WinHttpRequestState * set GeneratePackageOnBuild & ServicingVersion --- .../src/System.Net.Http.WinHttpHandler.csproj | 2 ++ .../System/Net/Http/WinHttpRequestState.cs | 22 +++++++++++++++++- .../System/Net/Http/WinHttpRequestStream.cs | 23 ++----------------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj b/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj index 404e965fd8f..21b2b54e3cb 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj @@ -4,6 +4,8 @@ true true true + true + 1 Provides a message handler for HttpClient based on the WinHTTP interface of Windows. While similar to HttpClientHandler, it provides developers more granular control over the application's HTTP communication than the HttpClientHandler. Commonly Used Types: diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs index a0af9a557f5..f8b01f140b7 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestState.cs @@ -157,6 +157,7 @@ public WinHttpTransportContext TransportContext public long CurrentBytesRead { get; set; } private GCHandle _cachedReceivePinnedBuffer; + private GCHandle _cachedSendPinnedBuffer; public void PinReceiveBuffer(byte[] buffer) { @@ -171,6 +172,19 @@ public void PinReceiveBuffer(byte[] buffer) } } + public void PinSendBuffer(byte[] buffer) + { + if (!_cachedSendPinnedBuffer.IsAllocated || _cachedSendPinnedBuffer.Target != buffer) + { + if (_cachedSendPinnedBuffer.IsAllocated) + { + _cachedSendPinnedBuffer.Free(); + } + + _cachedSendPinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned); + } + } + #region IDisposable Members private void Dispose(bool disposing) { @@ -193,12 +207,18 @@ private void Dispose(bool disposing) { // This method only gets called when the WinHTTP request handle is fully closed and thus all // async operations are done. So, it is safe at this point to unpin the buffers and release - // the strong GCHandle for this object. + // the strong GCHandle for the pinned buffers. if (_cachedReceivePinnedBuffer.IsAllocated) { _cachedReceivePinnedBuffer.Free(); _cachedReceivePinnedBuffer = default(GCHandle); } + + if (_cachedSendPinnedBuffer.IsAllocated) + { + _cachedSendPinnedBuffer.Free(); + _cachedSendPinnedBuffer = default(GCHandle); + } #if DEBUG Interlocked.Increment(ref s_dbg_operationHandleFree); #endif diff --git a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs index ebbb43eaeed..64288c537fa 100644 --- a/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs +++ b/src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpRequestStream.cs @@ -24,8 +24,6 @@ internal sealed class WinHttpRequestStream : Stream private readonly SafeWinHttpHandle _requestHandle; private readonly WinHttpChunkMode _chunkedMode; - private GCHandle _cachedSendPinnedBuffer; - internal WinHttpRequestStream(WinHttpRequestState state, WinHttpChunkMode chunkedMode) { _state = state; @@ -182,15 +180,7 @@ internal async Task EndUploadAsync(CancellationToken token) protected override void Dispose(bool disposing) { - if (!_disposed) - { - _disposed = true; - if (_cachedSendPinnedBuffer.IsAllocated) - { - _cachedSendPinnedBuffer.Free(); - } - } - + _disposed = true; base.Dispose(disposing); } @@ -234,16 +224,7 @@ private Task InternalWriteDataAsync(byte[] buffer, int offset, int count, { Debug.Assert(count > 0); - if (!_cachedSendPinnedBuffer.IsAllocated || _cachedSendPinnedBuffer.Target != buffer) - { - if (_cachedSendPinnedBuffer.IsAllocated) - { - _cachedSendPinnedBuffer.Free(); - } - - _cachedSendPinnedBuffer = GCHandle.Alloc(buffer, GCHandleType.Pinned); - } - + _state.PinSendBuffer(buffer); _state.TcsInternalWriteDataToRequestStream = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); From 49c1cf3c3653f34306203fd1ff00e1047ae54965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Thu, 16 May 2024 09:02:47 +0200 Subject: [PATCH 110/151] [wasm] Suppress export name minification (#102155) --- src/mono/wasm/build/WasmApp.Native.targets | 2 +- src/mono/wasm/wasm.proj | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 52b2988f8dd..7aa1203bd11 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -257,7 +257,7 @@ <_EmccLDFlags Include="$(EmccLinkOptimizationFlag)" /> <_EmccLDFlags Include="@(_EmccCommonFlags)" /> - <_EmccLDFlags Include="-s EXPORT_ES6=1" /> + <_EmccLDFlags Include="-s EXPORT_ES6=1 -lexports.js" /> <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> diff --git a/src/mono/wasm/wasm.proj b/src/mono/wasm/wasm.proj index 3b75fa9feeb..1790445cbc6 100644 --- a/src/mono/wasm/wasm.proj +++ b/src/mono/wasm/wasm.proj @@ -370,7 +370,8 @@ $(CMakeConfigurationEmccFlags) -s ASSERTIONS=1 -O2 - $(CMakeConfigurationLinkFlags) -s EXPORT_ES6=1 + + $(CMakeConfigurationLinkFlags) -s EXPORT_ES6=1 -lexports.js $(CMakeConfigurationLinkFlags) -msimd128 $(CMakeConfigurationLinkFlags) -Wno-pthreads-mem-growth $(CMakeConfigurationLinkFlags) --emit-symbol-map From 69c5ca890a80d925ba6bd4ca117db34eaa1bffd7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 16:34:23 -0600 Subject: [PATCH 111/151] [release/8.0-staging] Update dependencies from dotnet/source-build-reference-packages (#101917) * Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240501.1 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24163.3 -> To Version 8.0.0-alpha.1.24251.1 * Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240507.2 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24163.3 -> To Version 8.0.0-alpha.1.24257.2 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c2d1b0ca3f6..758d2328e9c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -99,9 +99,9 @@ a1cd44fdc64aa1f1c4630ddcd95580800d229180 - + https://github.com/dotnet/source-build-reference-packages - 79827eed138fd2575a8b24820b4f385ee4ffb6e6 + 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb From a7ed09cb748f3dec5c8028b8716ce41d3608b288 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 16:34:31 -0600 Subject: [PATCH 112/151] [release/8.0-staging] Update dependencies from dotnet/source-build-externals (#101979) * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240506.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24256.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240513.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24263.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240513.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24263.1 * Update dependencies from https://github.com/dotnet/source-build-externals build 20240513.1 Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24216.1 -> To Version 8.0.0-alpha.1.24263.1 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 758d2328e9c..f9d045a4d4a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb - + https://github.com/dotnet/source-build-externals - 908177a58a41532b3302c17f1e1a8cf1c1234545 + a3021ef9ed72d7bdf799092a47d2d024fc13bfcd From fe21c03db56cb2ab70de27b3f986e314e4a652c4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 May 2024 16:51:30 -0600 Subject: [PATCH 113/151] [release/8.0-staging] Update dependencies from dotnet/emsdk (#101941) * Update dependencies from https://github.com/dotnet/emsdk build 20240506.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24252.3 -> To Version 8.0.6-servicing.24256.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240507.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24252.3 -> To Version 8.0.6-servicing.24257.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240508.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24252.3 -> To Version 8.0.6-servicing.24258.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240514.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24252.3 -> To Version 8.0.6-servicing.24264.1 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 58277f76a14..3e394d788ef 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f9d045a4d4a..77cf3d794bd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - a1cd44fdc64aa1f1c4630ddcd95580800d229180 + 16d77ddacb12870344abbc4831387cc3e8f97168 - + https://github.com/dotnet/emsdk - a1cd44fdc64aa1f1c4630ddcd95580800d229180 + 16d77ddacb12870344abbc4831387cc3e8f97168 From 0abb5108159f9910dad02d2b76458ca3e35a7681 Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Thu, 16 May 2024 16:18:03 -0700 Subject: [PATCH 114/151] Enable repackaging of NETStandard 2.1 Targeting Pack (#102081) * Initial changes * Enable building in 8.0.6 release * Build netstandard 2.1 installer for RPM only * Update target build to 8.0.7 --- eng/Subsets.props | 1 + eng/Versions.props | 8 +++ src/installer/pkg/sfx/installers.proj | 4 ++ .../pkg/sfx/installers/netstandard2.1.proj | 60 +++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 src/installer/pkg/sfx/installers/netstandard2.1.proj diff --git a/eng/Subsets.props b/eng/Subsets.props index 69924723836..c27a2841277 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -507,6 +507,7 @@ + diff --git a/eng/Versions.props b/eng/Versions.props index ba8e464d4f3..677bef5639b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -260,4 +260,12 @@ 8.0.101 $(MicrosoftDotnetSdkInternalVersion) + + + true + diff --git a/src/installer/pkg/sfx/installers.proj b/src/installer/pkg/sfx/installers.proj index 7f4ce6b9c1c..2f947e6d9e2 100644 --- a/src/installer/pkg/sfx/installers.proj +++ b/src/installer/pkg/sfx/installers.proj @@ -22,6 +22,10 @@ + + + + + + true + false + netstandard-targeting-pack-2.1 + NETStandard.Library.Ref + NETStandard.Library.Ref 2.1.0 + false + ToolPack + true + netstandard-targeting-pack-2.1.0-x64.rpm + https://dotnetcli.blob.core.windows.net/dotnet/Runtime/3.1.0/$(OriginalNETStandard21PkgFilename) + $([MSBuild]::NormalizeDirectory('$(BaseIntermediateOutputPath)', 'download')) + $(NETStandard21TempDir)$(OriginalNETStandard21PkgFilename) + $(NETStandard21TempDir)netstandard21.semaphore + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 849596069fcb6ec0b30039ae05cadc9a40220ee6 Mon Sep 17 00:00:00 2001 From: Matous Kozak <55735845+matouskozak@users.noreply.github.com> Date: Fri, 17 May 2024 16:33:26 +0200 Subject: [PATCH 115/151] [mono] disable fullAOT-llvm x64 job (#102330) --- .../runtime-extra-platforms-other.yml | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml index ee481119ad9..e2e95c6bac5 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-other.yml @@ -355,40 +355,41 @@ jobs: # Mono CoreCLR runtime Test executions using live libraries and LLVM Full AOT # Only when Mono is changed # -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - buildConfig: Release - runtimeFlavor: mono - platforms: - - linux_x64 - # - linux_arm64 - variables: - - name: timeoutPerTestInMinutes - value: 60 - - name: timeoutPerTestCollectionInMinutes - value: 180 - jobParameters: - testGroup: innerloop - nameSuffix: AllSubsets_Mono_LLVMFullAot_RuntimeTests - runtimeVariant: llvmfullaot - buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true - timeoutInMinutes: 300 +# Disabled due to OOM errors: https://github.com/dotnet/runtime/issues/90427 +# - template: /eng/pipelines/common/platform-matrix.yml +# parameters: +# jobTemplate: /eng/pipelines/common/global-build-job.yml +# helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml +# buildConfig: Release +# runtimeFlavor: mono +# platforms: +# - linux_x64 +# # - linux_arm64 +# variables: +# - name: timeoutPerTestInMinutes +# value: 60 +# - name: timeoutPerTestCollectionInMinutes +# value: 180 +# jobParameters: +# testGroup: innerloop +# nameSuffix: AllSubsets_Mono_LLVMFullAot_RuntimeTests +# runtimeVariant: llvmfullaot +# buildArgs: -s mono+libs+clr.hosts+clr.iltools -c Release /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true +# timeoutInMinutes: 300 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), - eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), - eq(variables['isRollingBuild'], true)) - postBuildSteps: - - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml - parameters: - creator: dotnet-bot - llvmAotStepContainer: linux_x64_llvmaot - testRunNamePrefixSuffix: Mono_Release - extraVariablesTemplates: - - template: /eng/pipelines/common/templates/runtimes/test-variables.yml +# condition: >- +# or( +# eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), +# eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), +# eq(variables['isRollingBuild'], true)) +# postBuildSteps: +# - template: /eng/pipelines/common/templates/runtimes/build-runtime-tests-and-send-to-helix.yml +# parameters: +# creator: dotnet-bot +# llvmAotStepContainer: linux_x64_llvmaot +# testRunNamePrefixSuffix: Mono_Release +# extraVariablesTemplates: +# - template: /eng/pipelines/common/templates/runtimes/test-variables.yml # # Mono CoreCLR runtime Test executions using live libraries in interpreter mode From ba826e1e68a37031eb20ed427cf68e3090f3c23b Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Fri, 17 May 2024 12:51:12 -0700 Subject: [PATCH 116/151] Update branding to 8.0.7 (#102356) * Update branding to 8.0.7 * Update eng/Versions.props Co-authored-by: Larry Ewing --------- Co-authored-by: Larry Ewing --- eng/Versions.props | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index ba8e464d4f3..f5f01e6504e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,13 +1,13 @@ - 8.0.6 + 8.0.7 8 0 - 6 + 7 8.0.100 - 7.0.19 + 7.0.20 6.0.$([MSBuild]::Add($(PatchVersion),25)) servicing From 75dc9135818c14cafc8195b0b3b23f96f6e708f2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 13:48:20 -0700 Subject: [PATCH 117/151] Update dependencies from https://github.com/dotnet/emsdk build 20240514.1 (#102400) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24252.3 -> To Version 8.0.6-servicing.24264.1 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 58277f76a14..3e394d788ef 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c2d1b0ca3f6..3f51b32794a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - a1cd44fdc64aa1f1c4630ddcd95580800d229180 + 16d77ddacb12870344abbc4831387cc3e8f97168 - + https://github.com/dotnet/emsdk - a1cd44fdc64aa1f1c4630ddcd95580800d229180 + 16d77ddacb12870344abbc4831387cc3e8f97168 From a4902024b9733cb9d180d7d4db32825d9588bbbe Mon Sep 17 00:00:00 2001 From: Ilona Tomkowicz <32700855+ilonatommy@users.noreply.github.com> Date: Mon, 20 May 2024 18:03:58 +0000 Subject: [PATCH 118/151] Do not run n V8, only on the browser. (#102447) --- ...ystem.Globalization.Calendars.Hybrid.WASM.Tests.csproj | 8 ++++++++ .../Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj b/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj index 5b898363764..e70dcc3b08e 100644 --- a/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj +++ b/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj @@ -4,6 +4,14 @@ true true true + + + + WasmTestOnBrowser + $(TestArchiveRoot)browserornodejs/ + $(TestArchiveTestsRoot)$(OSPlatformConfig)/ + $(DefineConstants);TARGET_BROWSER + true diff --git a/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj b/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj index a39604c1cfa..0a0322d77f3 100644 --- a/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj +++ b/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj @@ -5,6 +5,14 @@ true true true + + + + WasmTestOnBrowser + $(TestArchiveRoot)browserornodejs/ + $(TestArchiveTestsRoot)$(OSPlatformConfig)/ + $(DefineConstants);TARGET_BROWSER + true From 7e758f66b96527ffb3f4e800bf5a9dcf5de9ca27 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 11:24:31 -0700 Subject: [PATCH 119/151] Update dependencies from https://github.com/dotnet/arcade build 20240516.3 (#102432) Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Archives , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.Build.Tasks.Packaging , Microsoft.DotNet.Build.Tasks.TargetFramework , Microsoft.DotNet.Build.Tasks.Templating , Microsoft.DotNet.Build.Tasks.Workloads , Microsoft.DotNet.CodeAnalysis , Microsoft.DotNet.GenAPI , Microsoft.DotNet.GenFacades , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.PackageTesting , Microsoft.DotNet.RemoteExecutor , Microsoft.DotNet.SharedFramework.Sdk , Microsoft.DotNet.VersionTools.Tasks , Microsoft.DotNet.XUnitConsoleRunner , Microsoft.DotNet.XUnitExtensions From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24266.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 72 +++++++++---------- eng/Versions.props | 30 ++++---- .../job/source-index-stage1.yml | 49 ++++++++----- .../templates/job/source-index-stage1.yml | 44 ++++++++---- global.json | 6 +- 5 files changed, 118 insertions(+), 83 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3f51b32794a..9e7077e92e7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -111,9 +111,9 @@ - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac @@ -121,69 +121,69 @@ 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac https://github.com/dotnet/runtime-assets @@ -334,9 +334,9 @@ https://github.com/dotnet/xharness 9d21162000c444b2da3d6cdd805d43e1af51453a - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac https://dev.azure.com/dnceng/internal/_git/dotnet-optimization diff --git a/eng/Versions.props b/eng/Versions.props index f5f01e6504e..fa91d8d68f3 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -87,21 +87,21 @@ 8.0.100 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 2.5.1-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 - 8.0.0-beta.24204.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 2.5.1-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 + 8.0.0-beta.24266.3 6.0.0-preview.1.102 diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f0513aee5b0..43ee0c202fc 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,15 +15,15 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml + - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -33,24 +34,23 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - os: windows + demands: ImageOverride -equals windows.vs2019.amd64 steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -62,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02d..43ee0c202fc 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -40,16 +41,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -61,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/global.json b/global.json index e0a39423109..4a9225b709f 100644 --- a/global.json +++ b/global.json @@ -8,9 +8,9 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24204.3", - "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24204.3", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.24266.3", + "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.24266.3", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "8.0.0-rc.1.23406.6" From f87df2cb38a5b2088c3f796bd5704be44d28961a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 12:41:41 -0700 Subject: [PATCH 120/151] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20240507.2 (#102431) Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.24163.3 -> To Version 8.0.0-alpha.1.24257.2 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9e7077e92e7..bd1feb57131 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -99,9 +99,9 @@ 16d77ddacb12870344abbc4831387cc3e8f97168 - + https://github.com/dotnet/source-build-reference-packages - 79827eed138fd2575a8b24820b4f385ee4ffb6e6 + 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb From 919e5b2b4995c0769f49d272042de50533b47c75 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 14:08:12 -0700 Subject: [PATCH 121/151] [release/8.0] Update dependencies from dotnet/emsdk (#102408) * Update dependencies from https://github.com/dotnet/emsdk build 20240517.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24264.1 -> To Version 8.0.7-servicing.24267.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240520.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24264.1 -> To Version 8.0.7-servicing.24270.2 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NuGet.config b/NuGet.config index 3e394d788ef..d74a62c8f12 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index bd1feb57131..aba57a86483 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 16d77ddacb12870344abbc4831387cc3e8f97168 + a3c7d8205559d673de8b7cdafd6d80d5f4d2cfed - + https://github.com/dotnet/emsdk - 16d77ddacb12870344abbc4831387cc3e8f97168 + a3c7d8205559d673de8b7cdafd6d80d5f4d2cfed diff --git a/eng/Versions.props b/eng/Versions.props index fa91d8d68f3..c723e90c1bf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -240,7 +240,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.6 + 8.0.7 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From 7650e52eed65b18feeb6c01b2f07228d1eafccc6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 14:44:55 -0700 Subject: [PATCH 122/151] Update dependencies from https://github.com/dotnet/source-build-externals build 20240519.1 (#102453) Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.24263.1 -> To Version 8.0.0-alpha.1.24269.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 77cf3d794bd..6d474020044 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -104,9 +104,9 @@ 6ed73280a6d70f7e7ac39c86f2abe8c10983f0bb - + https://github.com/dotnet/source-build-externals - a3021ef9ed72d7bdf799092a47d2d024fc13bfcd + 4f2151df120194f0268944f1b723c14820738fc8 From 3ee9306680508d33d56a7cb2bb101d97d0efed3b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 14:45:44 -0700 Subject: [PATCH 123/151] Update dependencies from https://github.com/dotnet/xharness build 20240429.2 (#102433) Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Common , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 8.0.0-prerelease.24208.4 -> To Version 8.0.0-prerelease.24229.2 Co-authored-by: dotnet-maestro[bot] --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index deb9fb7b21b..a42011c11af 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "8.0.0-prerelease.24208.4", + "version": "8.0.0-prerelease.24229.2", "commands": [ "xharness" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6d474020044..3a2af9bbaf8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -322,17 +322,17 @@ https://github.com/dotnet/runtime edbd5c769a19798b6955050baccf99e6797d3208 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 - + https://github.com/dotnet/xharness - 9d21162000c444b2da3d6cdd805d43e1af51453a + aacfb6328fdef17e572617bbb551431bb9cb1ff2 https://github.com/dotnet/arcade diff --git a/eng/Versions.props b/eng/Versions.props index 677bef5639b..f58fcfb6965 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -183,9 +183,9 @@ 1.1.0 17.4.0-preview-20220707-01 - 8.0.0-prerelease.24208.4 - 8.0.0-prerelease.24208.4 - 8.0.0-prerelease.24208.4 + 8.0.0-prerelease.24229.2 + 8.0.0-prerelease.24229.2 + 8.0.0-prerelease.24229.2 8.0.0-alpha.0.24229.2 2.4.2 1.0.0 From abbf9c6f7c9f1dab403eff742683e2c3e5dc29ac Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 May 2024 19:02:36 -0700 Subject: [PATCH 124/151] Update PGO to use the correct post-build steps model (#102527) --- eng/pipelines/runtime-official.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 6e6781bb134..d84f414f465 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -604,9 +604,10 @@ extends: buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO - extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml - extraStepsParameters: - name: PGO + postBuildSteps: + - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml + parameters: + name: PGO timeoutInMinutes: 95 # From 0be90100caf4dda8ebaa64a9c3abd1f4ec26299a Mon Sep 17 00:00:00 2001 From: Nikola Milosavljevic Date: Thu, 23 May 2024 09:31:54 -0700 Subject: [PATCH 125/151] Update condition for NETStandard 2.1 Targeting Pack build (#102507) --- eng/Versions.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index f58fcfb6965..26f979c5617 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -264,8 +264,8 @@ - true + true From 44d824898bb9b1f632887566131e7ef5b5ad1295 Mon Sep 17 00:00:00 2001 From: Ivan Povazan <55002338+ivanpovazan@users.noreply.github.com> Date: Fri, 24 May 2024 11:38:02 +0200 Subject: [PATCH 126/151] Remove linux-arm as supported RID for NativeAOT (#102072) Fixes https://github.com/dotnet/runtime/issues/100929 --- .../projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props | 1 - 1 file changed, 1 deletion(-) diff --git a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props index 1d51a78880a..7cbe74757e2 100644 --- a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props +++ b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props @@ -2,7 +2,6 @@ - From 984885ca9d99593dfbc23ce1a208e13bbb6a5c4e Mon Sep 17 00:00:00 2001 From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Fri, 24 May 2024 10:38:57 -0700 Subject: [PATCH 127/151] [release/8.0] Use V5 ESRP task with backing MI + AKV (#102582) --- .../coreclr/templates/sign-diagnostic-files.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml b/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml index 89cbbec6390..ed1f8523925 100644 --- a/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml +++ b/eng/pipelines/coreclr/templates/sign-diagnostic-files.yml @@ -12,10 +12,15 @@ steps: version: '6.0.x' installationPath: '$(Agent.TempDirectory)/dotnet' - - task: EsrpCodeSigning@1 + - task: EsrpCodeSigning@5 displayName: Sign Diagnostic Binaries inputs: - ConnectedServiceName: 'dotnetesrp-diagnostics-dnceng' + ConnectedServiceName: 'diagnostics-esrp-kvcertuser' + AppRegistrationClientId: '2234cdec-a13f-4bb2-aa63-04c57fd7a1f9' + AppRegistrationTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + AuthAKVName: 'clrdiag-esrp-id' + AuthCertName: 'dotnetesrp-diagnostics-aad-ssl-cert' + AuthSignCertName: 'dotnet-diagnostics-esrp-pki-onecert' FolderPath: ${{ parameters.basePath }} Pattern: | **/mscordaccore*.dll @@ -48,6 +53,7 @@ steps: SessionTimeout: ${{ parameters.timeoutInMinutes }} MaxConcurrency: '50' MaxRetryAttempts: '5' + PendingAnalysisWaitTimeoutMinutes: '5' env: DOTNET_MULTILEVEL_LOOKUP: 0 DOTNET_ROOT: '$(Agent.TempDirectory)/dotnet' From 48a9496667c8437db38177ba3f05f635bc189dfb Mon Sep 17 00:00:00 2001 From: Stephen Toub Date: Tue, 28 May 2024 21:41:04 -0400 Subject: [PATCH 128/151] [release/8.0] Fix cancellation unregistration in DataflowBlock.OutputAvailableAsync (#102376) * Fix cancellation unregistration in DataflowBlock.OutputAvailableAsync (#99632) OutputAvailableAsync is not unregistering from the supplied CancellationToken. If a cancelable token is supplied and is long lived, each call with that token to OutputAvailableAsync will add another callback into that token, and that will continue to grow until either the token is dropped or has been cancellation requested. For a long-lived cancellation token, this is akin to a leak. The implementation was trying to be too clever in avoiding an additional continuation that was previously there. However, this continuation makes it a lot easier to avoid possible deadlocks that can occur if a cancellation request comes in concurrently with a message being pushed. Instead of trying to avoid it, just use an async method, which still incurs the extra task but does so with less allocation and greatly simplifies the code while also fixing the issue, as all cleanup can now be done in the continuation as part of the async method. * Enable DataFlow package in servicing --------- Co-authored-by: Eric StJohn --- .../src/Base/DataflowBlock.cs | 131 +++++------------- .../System.Threading.Tasks.Dataflow.csproj | 2 + 2 files changed, 39 insertions(+), 94 deletions(-) diff --git a/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs b/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs index 63cdedf4b46..7007b0c80f5 100644 --- a/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs +++ b/src/libraries/System.Threading.Tasks.Dataflow/src/Base/DataflowBlock.cs @@ -1434,63 +1434,48 @@ public static Task OutputAvailableAsync(this ISourceBlock OutputAvailableAsync( this ISourceBlock source, CancellationToken cancellationToken) { - if (source is null) - { - throw new ArgumentNullException(nameof(source)); - } - - // Fast path for cancellation - if (cancellationToken.IsCancellationRequested) - return Common.CreateTaskFromCancellation(cancellationToken); - - // In a method like this, normally we would want to check source.Completion.IsCompleted - // and avoid linking completely by simply returning a completed task. However, - // some blocks that are completed still have data available, like WriteOnceBlock, - // which completes as soon as it gets a value and stores that value forever. - // As such, OutputAvailableAsync must link from the source so that the source - // can push data to us if it has it, at which point we can immediately unlink. + return + source is null ? throw new ArgumentNullException(nameof(source)) : + cancellationToken.IsCancellationRequested ? Common.CreateTaskFromCancellation(cancellationToken) : + Impl(source, cancellationToken); - // Create a target task that will complete when it's offered a message (but it won't accept the message) - var target = new OutputAvailableAsyncTarget(); - try + static async Task Impl(ISourceBlock source, CancellationToken cancellationToken) { - // Link from the source. If the source propagates a message during or immediately after linking - // such that our target is already completed, just return its task. - target._unlinker = source.LinkTo(target, DataflowLinkOptions.UnlinkAfterOneAndPropagateCompletion); + // In a method like this, normally we would want to check source.Completion.IsCompleted + // and avoid linking completely by simply returning a completed task. However, + // some blocks that are completed still have data available, like WriteOnceBlock, + // which completes as soon as it gets a value and stores that value forever. + // As such, OutputAvailableAsync must link from the source so that the source + // can push data to us if it has it, at which point we can immediately unlink. - // If the task is already completed (an exception may have occurred, or the source may have propagated - // a message to the target during LinkTo or soon thereafter), just return the task directly. - if (target.Task.IsCompleted) - { - return target.Task; - } + // Create a target task that will complete when it's offered a message (but it won't accept the message) + var target = new OutputAvailableAsyncTarget(); - // If cancellation could be requested, hook everything up to be notified of cancellation requests. - if (cancellationToken.CanBeCanceled) + // Link from the source. + using (source.LinkTo(target, DataflowLinkOptions.UnlinkAfterOneAndPropagateCompletion)) { - // When cancellation is requested, unlink the target from the source and cancel the target. - target._ctr = cancellationToken.Register( + CancellationTokenRegistration registration = default; + try + { + // Register for cancellation if the target isn't already completed (the source may have propagated + // a message to the target during LinkTo or soon thereafter). + if (!target.Task.IsCompleted) + { + registration = #if NET6_0_OR_GREATER - OutputAvailableAsyncTarget.CancelAndUnlink, + cancellationToken.UnsafeRegister(static (state, cancellationToken) => ((OutputAvailableAsyncTarget)state!).TrySetCanceled(cancellationToken), target); #else - static state => OutputAvailableAsyncTarget.CancelAndUnlink(state, default), + cancellationToken.Register(static state => ((OutputAvailableAsyncTarget)state!).TrySetCanceled(), target); #endif - target); - } - - return target.Task; - } - catch (Exception exc) - { - // Source.LinkTo could throw, as could cancellationToken.Register if cancellation was already requested - // such that it synchronously invokes the source's unlinker IDisposable, which could throw. - target.TrySetException(exc); - - // Undo the link from the source to the target - target.AttemptThreadSafeUnlink(); + } - // Return the now faulted task - return target.Task; + return await target.Task.ConfigureAwait(false); + } + finally + { + registration.Dispose(); + } + } } } @@ -1504,46 +1489,6 @@ public OutputAvailableAsyncTarget() : { } - /// - /// Cached continuation delegate that unregisters from cancellation and - /// marshals the antecedent's result to the return value. - /// - internal static readonly Func, object?, bool> s_handleCompletion = (antecedent, state) => - { - var target = state as OutputAvailableAsyncTarget; - Debug.Assert(target != null, "Expected non-null target"); - target._ctr.Dispose(); - return antecedent.GetAwaiter().GetResult(); - }; - - /// Cancels the target and unlinks the target from the source. - /// An OutputAvailableAsyncTarget. - /// The token that triggered cancellation - internal static void CancelAndUnlink(object? state, CancellationToken cancellationToken) - { - var target = state as OutputAvailableAsyncTarget; - Debug.Assert(target != null, "Expected a non-null target"); - - target.TrySetCanceled(cancellationToken); - target.AttemptThreadSafeUnlink(); - } - - /// Disposes of _unlinker if the target has been linked. - internal void AttemptThreadSafeUnlink() - { - // A race is possible. Therefore use an interlocked operation. - IDisposable? cachedUnlinker = _unlinker; - if (cachedUnlinker != null && Interlocked.CompareExchange(ref _unlinker, null, cachedUnlinker) == cachedUnlinker) - { - cachedUnlinker.Dispose(); - } - } - - /// The IDisposable used to unlink this target from its source. - internal IDisposable? _unlinker; - /// The registration used to unregister this target from the cancellation token. - internal CancellationTokenRegistration _ctr; - /// Completes the task when offered a message (but doesn't consume the message). DataflowMessageStatus ITargetBlock.OfferMessage(DataflowMessageHeader messageHeader, T messageValue, ISourceBlock? source, bool consumeToAccept) { @@ -1551,14 +1496,12 @@ DataflowMessageStatus ITargetBlock.OfferMessage(DataflowMessageHeader message if (source == null) throw new ArgumentNullException(nameof(source)); TrySetResult(true); + return DataflowMessageStatus.DecliningPermanently; } /// - void IDataflowBlock.Complete() - { - TrySetResult(false); - } + void IDataflowBlock.Complete() => TrySetResult(false); /// void IDataflowBlock.Fault(Exception exception) @@ -1572,13 +1515,13 @@ void IDataflowBlock.Fault(Exception exception) } /// - Task IDataflowBlock.Completion { get { throw new NotSupportedException(SR.NotSupported_MemberNotNeeded); } } + Task IDataflowBlock.Completion => throw new NotSupportedException(SR.NotSupported_MemberNotNeeded); /// The data to display in the debugger display attribute. private object DebuggerDisplayContent => $"{Common.GetNameForDebugger(this)} IsCompleted = {base.Task.IsCompleted}"; /// Gets the data to display in the debugger display attribute for this instance. - object IDebuggerDisplay.Content { get { return DebuggerDisplayContent; } } + object IDebuggerDisplay.Content => DebuggerDisplayContent; } #endregion diff --git a/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj b/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj index 9835aa30e3b..3a4156130e8 100644 --- a/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj +++ b/src/libraries/System.Threading.Tasks.Dataflow/src/System.Threading.Tasks.Dataflow.csproj @@ -2,6 +2,8 @@ $(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.1;netstandard2.0;$(NetFrameworkMinimum) true + true + 1 TPL Dataflow promotes actor/agent-oriented designs through primitives for in-process message passing, dataflow, and pipelining. TDF builds upon the APIs and scheduling infrastructure provided by the Task Parallel Library (TPL), and integrates with the language support for asynchrony provided by C#, Visual Basic, and F#. Commonly Used Types: From 7b08fa8922f90028efd64f2d3a1fbb51165d07e3 Mon Sep 17 00:00:00 2001 From: Vlad Brezae Date: Wed, 29 May 2024 11:00:49 +0300 Subject: [PATCH 129/151] [mono][interp] Keep delegate alive during invocation (#100832) (#102159) When invoking a delegate, we were overwritting the stack slot containing the delegate object reference. In the case of invoking a delegate for a dynamic method, we were running into issues when the delegate object is collected while the method is executed because the method code is also discarded. --- src/mono/mono/mini/interp/interp-internals.h | 1 + src/mono/mono/mini/interp/interp.c | 3 +++ src/mono/mono/mini/interp/transform.c | 21 ++++++++++++++++++++ src/mono/mono/mini/interp/transform.h | 1 + 4 files changed, 26 insertions(+) diff --git a/src/mono/mono/mini/interp/interp-internals.h b/src/mono/mono/mini/interp/interp-internals.h index 4e0be7db043..79b93dae1fb 100644 --- a/src/mono/mono/mini/interp/interp-internals.h +++ b/src/mono/mono/mini/interp/interp-internals.h @@ -153,6 +153,7 @@ struct InterpMethod { unsigned int hasthis; // boolean MonoProfilerCallInstrumentationFlags prof_flags; InterpMethodCodeType code_type; + int ref_slot_offset; // GC visible pointer slot #ifdef ENABLE_EXPERIMENT_TIERED MiniTieredCounter tiered_counter; #endif diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index 620dc135d50..2942fb23d3f 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -4047,6 +4047,9 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause } cmethod = del_imethod; if (!is_multicast) { + int ref_slot_offset = frame->imethod->ref_slot_offset; + if (ref_slot_offset >= 0) + LOCAL_VAR (ref_slot_offset, gpointer) = del; if (cmethod->param_count == param_count + 1) { // Target method is static but the delegate has a target object. We handle // this separately from the case below, because, for these calls, the instance diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index 0d5ef5380de..0150c3d32e3 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -412,6 +412,17 @@ create_interp_dummy_var (TransformData *td) td->locals [td->dummy_var].flags = INTERP_LOCAL_FLAG_GLOBAL; } +static int alloc_global_var_offset (TransformData *td, int var); + +static void +interp_create_ref_handle_var (TransformData *td) +{ + int var = create_interp_local_explicit (td, m_class_get_byval_arg (mono_defaults.int_class), sizeof (gpointer)); + td->locals [var].flags = INTERP_LOCAL_FLAG_GLOBAL; + alloc_global_var_offset (td, var); + td->ref_handle_var = var; +} + static int get_tos_offset (TransformData *td) { @@ -3798,6 +3809,10 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target td->last_ins->data [0] = get_data_item_index_imethod (td, mono_interp_get_imethod (target_method)); } else { if (is_delegate_invoke) { + // MINT_CALL_DELEGATE will store the delegate object into this slot so it is kept alive + // while the method is invoked + if (td->ref_handle_var == -1) + interp_create_ref_handle_var (td); interp_add_ins (td, MINT_CALL_DELEGATE); interp_ins_set_dreg (td->last_ins, dreg); interp_ins_set_sreg (td->last_ins, MINT_CALL_ARGS_SREG); @@ -11149,6 +11164,7 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG td->n_data_items = 0; td->max_data_items = 0; td->dummy_var = -1; + td->ref_handle_var = -1; td->data_items = NULL; td->data_hash = g_hash_table_new (NULL, NULL); #ifdef ENABLE_EXPERIMENT_TIERED @@ -11287,6 +11303,11 @@ generate (MonoMethod *method, MonoMethodHeader *header, InterpMethod *rtm, MonoG mono_interp_register_imethod_data_items (rtm->data_items, td->imethod_items); rtm->patchpoint_data = td->patchpoint_data; + if (td->ref_handle_var != -1) + rtm->ref_slot_offset = td->locals [td->ref_handle_var].offset; + else + rtm->ref_slot_offset = -1; + /* Save debug info */ interp_save_debug_info (rtm, header, td, td->line_numbers); diff --git a/src/mono/mono/mini/interp/transform.h b/src/mono/mono/mini/interp/transform.h index 2bfbe423905..c9ff2df80a5 100644 --- a/src/mono/mono/mini/interp/transform.h +++ b/src/mono/mono/mini/interp/transform.h @@ -227,6 +227,7 @@ typedef struct gint32 max_stack_size; InterpLocal *locals; int dummy_var; + int ref_handle_var; int *local_ref_count; unsigned int il_locals_offset; unsigned int il_locals_size; From 32400f6b367f3ac34182c00710a62a19a9ef2161 Mon Sep 17 00:00:00 2001 From: Eduardo Velarde <32459232+eduardo-vp@users.noreply.github.com> Date: Wed, 29 May 2024 11:10:33 -0700 Subject: [PATCH 130/151] [release/8.0-staging] Backport 1ES templates (#102681) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [mono][infra] Fix mono-aot-cross build for linux-arm64 using biarch image (#91019) * Use host rootfs when TARGET_BUILD_ARCH matches * Don't build libraries native packages in the PGO leg (#92729) * Reenable OneLocBuild in runtime-official.yml (#95014) * Move FreeBSD CI leg to CBL-Mariner and v13 (#97038) * Use Ubuntu-22.04-based Tizen image (#98589) * Start moving over to official templates (#99433) * Start moving over to official templates * Parameterize runtime-inner-loop * Use default false for isOfficialBuild in pipeline-with-resources * Respond to PR comments * Code review comments * Use default template args * Move linux_x86, tizen_armel, and freebsd_x64 jobs to global-build.yml * Update thunktemplates.S (#100066) * Import pool providers (#100851) Import pool-providers in common variables. This should allow all stages access to the pool provider variables. * Pass templatePath to eng/pipelines/common/templates/runtimes/xplat-job.yml --------- Co-authored-by: Milos Kotlar Co-authored-by: Jeremy Koritzinsky Co-authored-by: Alexander Köplinger Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com> Co-authored-by: Andy Gocke Co-authored-by: Filip Navara Co-authored-by: Eduardo Manuel Velarde Polar --- eng/pipelines/common/global-build-job.yml | 30 ++- .../templates/pipeline-with-resources.yml | 231 +++++++++--------- .../templates/publish-build-artifacts.yml | 22 ++ .../templates/publish-pipeline-artifacts.yml | 17 ++ .../templates/runtimes/build-test-job.yml | 18 +- .../common/templates/runtimes/xplat-job.yml | 3 +- .../common/templates/template1es.yml | 31 +++ .../common/templates/templateDispatch.yml | 13 + .../common/templates/templatePublic.yml | 21 ++ eng/pipelines/common/upload-artifact-step.yml | 15 +- .../upload-intermediate-artifacts-step.yml | 14 +- eng/pipelines/common/variables.yml | 7 + eng/pipelines/common/xplat-setup.yml | 15 +- .../coreclr/templates/xplat-pipeline-job.yml | 2 + eng/pipelines/global-build.yml | 95 +++++++ .../mono/templates/generate-offsets.yml | 30 ++- .../mono/templates/workloads-build.yml | 18 +- .../mono/templates/xplat-pipeline-job.yml | 2 + .../jobs/prepare-signed-artifacts.yml | 17 +- eng/pipelines/official/stages/publish.yml | 6 +- eng/pipelines/runtime-official.yml | 46 +++- eng/pipelines/runtime.yml | 47 +--- src/coreclr/debug/daccess/CMakeLists.txt | 2 +- src/coreclr/vm/arm/thunktemplates.S | 2 +- src/mono/mono.proj | 12 +- 25 files changed, 478 insertions(+), 238 deletions(-) create mode 100644 eng/pipelines/common/templates/publish-build-artifacts.yml create mode 100644 eng/pipelines/common/templates/publish-pipeline-artifacts.yml create mode 100644 eng/pipelines/common/templates/template1es.yml create mode 100644 eng/pipelines/common/templates/templateDispatch.yml create mode 100644 eng/pipelines/common/templates/templatePublic.yml diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index 4b06818b140..86cea9fbd98 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -30,9 +30,10 @@ parameters: extraVariablesTemplates: [] isManualCodeQLBuild: false preBuildSteps: [] + templatePath: 'templates' jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: ${{ if eq(parameters.hostedOs, '') }}: name: ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.nameSuffix) }} @@ -141,6 +142,7 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} + steps: - ${{ if eq(parameters.osGroup, 'windows') }}: - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml @@ -181,7 +183,7 @@ jobs: path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - ${{ if eq(parameters.isSourceBuild, true) }}: - - template: /eng/common/templates/steps/source-build.yml + - template: /eng/common/${{ parameters.templatePath }}/steps/source-build.yml parameters: platform: baseOS: ${{ parameters.baseOS }} @@ -285,14 +287,16 @@ jobs: displayName: Collect vslogs on exit condition: always() - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' - PublishLocation: Container - ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: - ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/' + PublishLocation: Container + ${{ if notin(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.osSubGroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + ${{ if in(parameters.osGroup, 'browser', 'wasi') }}: + ArtifactName: Logs_Build_Attempt$(System.JobAttempt)_${{ parameters.osGroup }}_${{ parameters.archType }}_${{ parameters.hostedOs }}_${{ parameters.buildConfig }}_${{ parameters.nameSuffix }} + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/pipeline-with-resources.yml b/eng/pipelines/common/templates/pipeline-with-resources.yml index edba5767b61..f5fd4abbd25 100644 --- a/eng/pipelines/common/templates/pipeline-with-resources.yml +++ b/eng/pipelines/common/templates/pipeline-with-resources.yml @@ -1,113 +1,124 @@ parameters: - name: stages type: stageList - -resources: - containers: - - container: linux_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_armv6 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 - env: - ROOTFS_DIR: /crossrootfs/armv6 - - - container: linux_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64 - env: - ROOTFS_DIR: /crossrootfs/arm64 - - - container: linux_musl_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_musl_arm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine - env: - ROOTFS_DIR: /crossrootfs/arm - - - container: linux_musl_arm64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine - env: - ROOTFS_DIR: /crossrootfs/arm64 - - # This container contains all required toolsets to build for Android and for Linux with bionic libc. - - container: linux_bionic - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 - - # This container contains all required toolsets to build for Android as well as tooling to build docker images. - - container: android_docker - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker - - - container: linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: linux_x86 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 - env: - ROOTFS_DIR: /crossrootfs/x86 - - - container: linux_x64_dev_innerloop - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 - - # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. - - container: SourceBuild_centos_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. - - container: SourceBuild_linux_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build - - - container: linux_s390x - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x - env: - ROOTFS_DIR: /crossrootfs/s390x - - - container: linux_ppc64le - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le - env: - ROOTFS_DIR: /crossrootfs/ppc64le - - - container: linux_riscv64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 - env: - ROOTFS_DIR: /crossrootfs/riscv64 - - - container: debian-12-gcc13-amd64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 - - - container: linux_x64_llvmaot - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - - - container: browser_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: wasi_wasm - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: freebsd_x64 - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-freebsd-12 - env: - ROOTFS_DIR: /crossrootfs/x64 - - - container: tizen_armel - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-armel-tizen - env: - ROOTFS_DIR: /crossrootfs/armel - - - container: debpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg - - - container: rpmpkg - image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm - -stages: ${{ parameters.stages }} + - name: isOfficialBuild + type: boolean + default: false + +extends: + template: templateDispatch.yml + parameters: + ${{ if parameters.isOfficialBuild }}: + templatePath: template1es.yml + ${{ else }}: + templatePath: templatePublic.yml + + stages: ${{ parameters.stages }} + + containers: + linux_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_armv6: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-cross-armv6-raspbian-10 + env: + ROOTFS_DIR: /crossrootfs/armv6 + + linux_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-biarch-amd64-arm64 + env: + ROOTFS_HOST_DIR: /crossrootfs/x64 + ROOTFS_DIR: /crossrootfs/arm64 + + linux_musl_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-alpine + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_musl_arm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm-alpine + env: + ROOTFS_DIR: /crossrootfs/arm + + linux_musl_arm64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-arm64-alpine + env: + ROOTFS_DIR: /crossrootfs/arm64 + + # This container contains all required toolsets to build for Android and for Linux with bionic libc. + linux_bionic: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-android-amd64 + + # This container contains all required toolsets to build for Android as well as tooling to build docker images. + android_docker: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-android-docker + + linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64 + env: + ROOTFS_DIR: /crossrootfs/x64 + + linux_x86: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-x86 + env: + ROOTFS_DIR: /crossrootfs/x86 + + linux_x64_dev_innerloop: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04 + + # We use a CentOS Stream 9 image here to test building from source on CentOS Stream 9. + SourceBuild_centos_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + # AlmaLinux 8 is a RHEL 8 rebuild, so we use it to test building from source on RHEL 8. + SourceBuild_linux_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:almalinux-8-source-build + + linux_s390x: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-s390x + env: + ROOTFS_DIR: /crossrootfs/s390x + + linux_ppc64le: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-cross-ppc64le + env: + ROOTFS_DIR: /crossrootfs/ppc64le + + linux_riscv64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-riscv64 + env: + ROOTFS_DIR: /crossrootfs/riscv64 + + debian-12-gcc13-amd64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-gcc13-amd64 + + linux_x64_llvmaot: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + + browser_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + wasi_wasm: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-webassembly-20230913040940-1edc1c6 + env: + ROOTFS_DIR: /crossrootfs/x64 + + freebsd_x64: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-cross-amd64-freebsd-13 + env: + ROOTFS_DIR: /crossrootfs/x64 + + tizen_armel: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-cross-armel-tizen + env: + ROOTFS_DIR: /crossrootfs/armel + + debpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-22.04-debpkg + + rpmpkg: + image: mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-2.0-fpm \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-build-artifacts.yml b/eng/pipelines/common/templates/publish-build-artifacts.yml new file mode 100644 index 00000000000..b9b263c361f --- /dev/null +++ b/eng/pipelines/common/templates/publish-build-artifacts.yml @@ -0,0 +1,22 @@ +parameters: + - name: isOfficialBuild + type: boolean + - name: displayName + type: string + - name: inputs + type: object + - name: condition + type: string + default: '' + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} + - ${{ else }}: + - task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/publish-pipeline-artifacts.yml b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml new file mode 100644 index 00000000000..81f292ec552 --- /dev/null +++ b/eng/pipelines/common/templates/publish-pipeline-artifacts.yml @@ -0,0 +1,17 @@ +parameters: +- name: displayName + type: string +- name: inputs + type: object +- name: isOfficialBuild + type: boolean + +steps: + - ${{ if parameters.isOfficialBuild }}: + - task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} + - ${{ else }}: + - task: PublishPipelineArtifact@1 + displayName: ${{ parameters.displayName }} + inputs: ${{ parameters.inputs }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/runtimes/build-test-job.yml b/eng/pipelines/common/templates/runtimes/build-test-job.yml index e249e8ac922..2809d148770 100644 --- a/eng/pipelines/common/templates/runtimes/build-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -14,6 +14,7 @@ parameters: dependsOn: [] dependOnEvaluatePaths: false crossBuild: false + isOfficialBuild: false ### Build managed test components (native components are getting built as part ### of the product build job). @@ -142,12 +143,13 @@ jobs: artifactName: $(microsoftNetSdkIlArtifactName) displayName: 'Microsoft.NET.Sdk.IL package' - # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + ArtifactName: '${{ parameters.runtimeFlavor }}_Common_Runtime_TestBuildLogs_Attempt$(System.JobAttempt)_AnyOS_AnyCPU_$(buildConfig)_${{ parameters.testGroup }}' + continueOnError: true + condition: always() diff --git a/eng/pipelines/common/templates/runtimes/xplat-job.yml b/eng/pipelines/common/templates/runtimes/xplat-job.yml index 23e74c70e57..625d88d63d3 100644 --- a/eng/pipelines/common/templates/runtimes/xplat-job.yml +++ b/eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -20,11 +20,12 @@ parameters: enableMicrobuild: '' gatherAssetManifests: false disableComponentGovernance: false + templatePath: 'templates' variables: {} ## any extra variables to add to the defaults defined below jobs: -- template: /eng/common/templates/job/job.yml +- template: /eng/common/${{ parameters.templatePath }}/job/job.yml parameters: name: ${{ parameters.name }} diff --git a/eng/pipelines/common/templates/template1es.yml b/eng/pipelines/common/templates/template1es.yml new file mode 100644 index 00000000000..0770e37d6bd --- /dev/null +++ b/eng/pipelines/common/templates/template1es.yml @@ -0,0 +1,31 @@ + + +parameters: + - name: templatePath + type: string + default: 'templates-official' + - name: stages + type: stageList + - name: containers + type: object + + +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + containers: + ${{ parameters.containers }} + + stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templateDispatch.yml b/eng/pipelines/common/templates/templateDispatch.yml new file mode 100644 index 00000000000..1860af47aee --- /dev/null +++ b/eng/pipelines/common/templates/templateDispatch.yml @@ -0,0 +1,13 @@ +parameters: + - name: templatePath + type: string + - name: stages + type: stageList + - name: containers + type: object + +extends: + template: ${{ parameters.templatePath }} + parameters: + stages: ${{ parameters.stages }} + containers: ${{ parameters.containers }} \ No newline at end of file diff --git a/eng/pipelines/common/templates/templatePublic.yml b/eng/pipelines/common/templates/templatePublic.yml new file mode 100644 index 00000000000..cd7c0272016 --- /dev/null +++ b/eng/pipelines/common/templates/templatePublic.yml @@ -0,0 +1,21 @@ + +parameters: + - name: templatePath + type: string + default: 'templates' + - name: stages + type: stageList + - name: containers + type: object + +resources: + containers: + - ${{ each container_pair in parameters.containers }}: + - ${{ if container_pair.value.image }}: + - container: ${{ container_pair.key }} + ${{ each pair in container_pair.value }}: + ${{ if notIn(pair.key, 'tenantId', 'identityType', 'registry') }}: + ${{ pair.key }}: ${{ pair.value }} + + +stages: ${{ parameters.stages }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-artifact-step.yml b/eng/pipelines/common/upload-artifact-step.yml index 249da066c7a..d4091a7cc19 100644 --- a/eng/pipelines/common/upload-artifact-step.yml +++ b/eng/pipelines/common/upload-artifact-step.yml @@ -7,6 +7,7 @@ parameters: artifactName: '' displayName: '' condition: succeeded() + isOfficialBuild: false steps: # Zip Artifact @@ -20,9 +21,11 @@ steps: includeRootFolder: ${{ parameters.includeRootFolder }} condition: ${{ parameters.condition }} - - task: PublishBuildArtifacts@1 - displayName: 'Publish ${{ parameters.displayName }}' - inputs: - pathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} - artifactName: ${{ parameters.artifactName }} - condition: ${{ parameters.condition }} + - template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: ${{ parameters.isOfficialBuild }} + displayName: 'Publish ${{ parameters.displayName }}' + inputs: + PathtoPublish: $(Build.StagingDirectory)/${{ parameters.artifactName }}${{ parameters.archiveExtension }} + artifactName: ${{ parameters.artifactName }} + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/pipelines/common/upload-intermediate-artifacts-step.yml b/eng/pipelines/common/upload-intermediate-artifacts-step.yml index bde6c61a0a0..da9b1ef0b62 100644 --- a/eng/pipelines/common/upload-intermediate-artifacts-step.yml +++ b/eng/pipelines/common/upload-intermediate-artifacts-step.yml @@ -25,9 +25,11 @@ steps: TargetFolder: '$(Build.StagingDirectory)/IntermediateArtifacts/${{ parameters.name }}' CleanTargetFolder: true -- task: PublishBuildArtifacts@1 - displayName: Publish intermediate artifacts - inputs: - pathToPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' - artifactName: IntermediateArtifacts - artifactType: container +- template: /eng/pipelines/common/templates/publish-build-artifacts.yml + parameters: + isOfficialBuild: true + displayName: Publish intermediate artifacts + inputs: + PathtoPublish: '$(Build.StagingDirectory)/IntermediateArtifacts' + ArtifactName: IntermediateArtifacts + ArtifactType: container diff --git a/eng/pipelines/common/variables.yml b/eng/pipelines/common/variables.yml index 2a757f82572..075dea178ab 100644 --- a/eng/pipelines/common/variables.yml +++ b/eng/pipelines/common/variables.yml @@ -1,3 +1,8 @@ +parameters: + - name: templatePath + type: string + default: 'templates' + variables: # These values enable longer delays, configurable number of retries, and special understanding of TCP hang-up @@ -54,3 +59,5 @@ variables: eq(variables['isRollingBuild'], true))) ] - template: /eng/pipelines/common/perf-variables.yml + +- template: /eng/common/${{ parameters.templatePath }}/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 675a2679201..794a23bb218 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -22,7 +22,7 @@ jobs: dependOnEvaluatePaths: ${{ and(eq(variables['Build.Reason'], 'PullRequest'), in(variables['Build.DefinitionName'], 'runtime', 'runtime-community', 'runtime-extra-platforms', 'runtime-wasm', 'runtime-wasm-libtests', 'runtime-wasm-non-libtests', 'dotnet-linker-tests', 'runtime-dev-innerloop', 'runtime-coreclr superpmi-replay', 'runtime-coreclr superpmi-diffs')) }} variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/${{ coalesce(parameters.jobParameters.templatePath, 'templates') }}/variables/pool-providers.yml # Disable component governance in our CI builds. These builds are not shipping nor # are they a service. Also the component governance jobs issue lots of inconsequential # warnings and errors into our build timelines that make it hard to track down @@ -168,12 +168,19 @@ jobs: # Official Build Linux Pool ${{ if and(or(in(parameters.osGroup, 'linux', 'freebsd', 'android', 'tizen'), eq(parameters.jobParameters.hostedOs, 'linux')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals 1es-ubuntu-2204 + os: linux - # OSX Build Pool (we don't have on-prem OSX BuildPool). - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + # OSX Public Build Pool (we don't have on-prem OSX BuildPool). + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), eq(variables['System.TeamProject'], 'public')) }}: vmImage: 'macos-12' + # OSX Internal Pool + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), ne(variables['System.TeamProject'], 'public')) }}: + name: "Azure Pipelines" + vmImage: 'macOS-12' + os: macOS + # Official Build Windows Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) diff --git a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml index b6c57be0c7f..82d6346a60d 100644 --- a/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/coreclr/templates/xplat-pipeline-job.yml @@ -11,6 +11,7 @@ parameters: liveLibrariesBuildConfig: '' strategy: '' pool: '' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/global-build.yml b/eng/pipelines/global-build.yml index 6f80e6ad104..8b6291cd0db 100644 --- a/eng/pipelines/global-build.yml +++ b/eng/pipelines/global-build.yml @@ -30,6 +30,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: @@ -177,3 +178,97 @@ extends: timeoutInMinutes: 95 condition: eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true) + + # + # Build CoreCLR as a non-portable build + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - tizen_armel + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NonPortable + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR with no R2R + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml + buildConfig: checked + runtimeFlavor: coreclr + platforms: + - linux_x86 + jobParameters: + testScope: innerloop + nameSuffix: CoreCLR_NoR2R + buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) + timeoutInMinutes: 120 + condition: >- + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + # + # Build CoreCLR release + # Always as they are needed by Installer and we always build and test the Installer. + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: release + platforms: + - freebsd_x64 + jobParameters: + testGroup: innerloop + # Mono/runtimetests also need this, but skip for wasm + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + platforms: + - freebsd_x64 + jobParameters: + testScope: innerloop + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_installer.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true), + eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/installer/jobs/build-job.yml + buildConfig: Release + platforms: + - freebsd_x64 + jobParameters: + liveRuntimeBuildConfig: release + liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} + runOnlyIfDependenciesSucceeded: true + condition: + or( + eq(dependencies.evaluate_paths.outputs['SetPathVars_non_mono_and_wasm.containsChange'], true), + eq(variables['isRollingBuild'], true)) diff --git a/eng/pipelines/mono/templates/generate-offsets.yml b/eng/pipelines/mono/templates/generate-offsets.yml index c68adfe67a9..909ca9a99c5 100644 --- a/eng/pipelines/mono/templates/generate-offsets.yml +++ b/eng/pipelines/mono/templates/generate-offsets.yml @@ -9,11 +9,13 @@ parameters: pool: '' condition: true isOfficialBuild: false + templatePath: 'templates' ### Product build jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} osGroup: ${{ parameters.osGroup }} osSubGroup: ${{ parameters.osSubGroup }} @@ -76,17 +78,21 @@ jobs: contents: '**/offsets-*.h' targetFolder: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles/' - - task: PublishPipelineArtifact@1 - displayName: Upload offset files - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' - artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Upload offset files + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles' + artifactName: 'Mono_Offsets_$(osGroup)$(osSubGroup)' # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'BuildLogs_Attempt$(System.JobAttempt)_Mono_Offsets_$(osGroup)$(osSubGroup)' + continueOnError: true + condition: always() diff --git a/eng/pipelines/mono/templates/workloads-build.yml b/eng/pipelines/mono/templates/workloads-build.yml index 4a50ed665e2..a10bf343fa4 100644 --- a/eng/pipelines/mono/templates/workloads-build.yml +++ b/eng/pipelines/mono/templates/workloads-build.yml @@ -12,11 +12,13 @@ parameters: runtimeVariant: '' testGroup: '' timeoutInMinutes: '' + templatePath: 'templates' variables: {} jobs: - template: xplat-pipeline-job.yml parameters: + templatePath: ${{ parameters.templatePath }} archType: ${{ parameters.archType }} buildConfig: ${{ parameters.buildConfig }} container: ${{ parameters.container }} @@ -92,13 +94,15 @@ jobs: name: workloads # Publish Logs - - task: PublishPipelineArtifact@1 - displayName: Publish Logs - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' - continueOnError: true - condition: always() + - template: /eng/pipelines/common/templates/publish-pipeline-artifacts.yml + parameters: + displayName: Publish Logs + isOfficialBuild: ${{ parameters.isOfficialBuild }} + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'WorkloadLogs_Attempt$(System.JobAttempt)' + continueOnError: true + condition: always() # Delete wixpdb files before they are uploaded to artifacts - task: DeleteFiles@1 diff --git a/eng/pipelines/mono/templates/xplat-pipeline-job.yml b/eng/pipelines/mono/templates/xplat-pipeline-job.yml index 1ca84d9caac..67b43722e0c 100644 --- a/eng/pipelines/mono/templates/xplat-pipeline-job.yml +++ b/eng/pipelines/mono/templates/xplat-pipeline-job.yml @@ -12,6 +12,7 @@ parameters: pool: '' runtimeVariant: '' liveRuntimeBuildConfig: 'release' + templatePath: 'templates' # arcade-specific parameters condition: true @@ -28,6 +29,7 @@ parameters: jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml parameters: + templatePath: ${{ parameters.templatePath }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml index 908f2b64c71..24fd2df48d7 100644 --- a/eng/pipelines/official/jobs/prepare-signed-artifacts.yml +++ b/eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -20,6 +20,14 @@ jobs: - name: SignType value: $[ coalesce(variables.OfficialSignType, 'real') ] + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish BuildLogs' + condition: succeededOrFailed() + targetPath: '$(Build.StagingDirectory)\BuildLogs' + artifactName: ${{ parameters.logArtifactName }} + steps: - checkout: self clean: true @@ -65,11 +73,4 @@ jobs: **/*.binlog TargetFolder: '$(Build.StagingDirectory)\BuildLogs' continueOnError: true - condition: succeededOrFailed() - - - task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.logArtifactName }} - condition: succeededOrFailed() + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/pipelines/official/stages/publish.yml b/eng/pipelines/official/stages/publish.yml index d23afa7003b..9553baae305 100644 --- a/eng/pipelines/official/stages/publish.yml +++ b/eng/pipelines/official/stages/publish.yml @@ -7,7 +7,7 @@ stages: - stage: PrepareForPublish displayName: Prepare for Publish variables: - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml jobs: # Prep artifacts: sign them and upload pipeline artifacts expected by stages-based publishing. - template: /eng/pipelines/official/jobs/prepare-signed-artifacts.yml @@ -15,7 +15,7 @@ stages: PublishRidAgnosticPackagesFromPlatform: ${{ parameters.PublishRidAgnosticPackagesFromPlatform }} # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - - template: /eng/common/templates/job/publish-build-assets.yml + - template: /eng/common/templates-official/job/publish-build-assets.yml parameters: publishUsingPipelines: true publishAssetsImmediately: true @@ -26,7 +26,7 @@ stages: symbolPublishingAdditionalParameters: '/p:PublishSpecialClrFiles=true' # Stages-based publishing entry point -- template: /eng/common/templates/post-build/post-build.yml +- template: /eng/common/templates-official/post-build/post-build.yml parameters: publishingInfraVersion: ${{ parameters.publishingInfraVersion }} validateDependsOn: diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index d84f414f465..7cadf6a800f 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -23,6 +23,8 @@ pr: none variables: - template: /eng/pipelines/common/variables.yml + parameters: + templatePath: 'templates-official' - template: /eng/pipelines/common/internal-variables.yml parameters: teamName: dotnet-core-acquisition @@ -33,6 +35,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: true stages: - stage: Build jobs: @@ -41,19 +44,18 @@ extends: # Localization build # - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/8.0') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: runtime - MirrorBranch: release/8.0 - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-RUNTIME' + - template: /eng/common/templates-official/job/onelocbuild.yml + parameters: + MirrorRepo: runtime + MirrorBranch: main + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-RUNTIME' # # Source Index Build # - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates/job/source-index-stage1.yml + - template: /eng/common/templates-official/job/source-index-stage1.yml parameters: sourceIndexBuildCommand: build.cmd -subset libs.sfx+libs.oob -binarylog -os linux -ci /p:SkipLibrariesNativeRuntimePackages=true @@ -70,6 +72,7 @@ extends: - windows_x64 - windows_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -105,6 +108,7 @@ extends: platforms: - windows_x86 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -138,6 +142,7 @@ extends: - osx_arm64 - osx_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.nativeaotruntime+host.native -c $(_BuildConfig) /bl:$(Build.SourcesDirectory)/artifacts/logs/$(_BuildConfig)/CoreClrNativeBuild.binlog nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -197,6 +202,7 @@ extends: - linux_musl_arm - linux_musl_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.runtime+clr.alljits+clr.corelib+clr.nativecorelib+clr.tools+clr.aot+clr.packages+libs+host+packs -c $(_BuildConfig) nameSuffix: CoreCLR isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -209,12 +215,12 @@ extends: SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig) Contents: libcoreclr.so TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch) - - task: PublishBuildArtifacts@1 + - task: 1ES.PublishBuildArtifacts@1 displayName: Publish runtime for CrossDac inputs: - pathToPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts + PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts PublishLocation: Container - artifactName: CoreCLRCrossDacArtifacts + ArtifactName: CoreCLRCrossDacArtifacts # Create RPMs and DEBs - template: /eng/pipelines/installer/jobs/steps/build-linux-package.yml parameters: @@ -250,6 +256,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath) nameSuffix: CrossDac isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -320,6 +327,7 @@ extends: - linux_bionic_arm64 - linux_bionic_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c $(_BuildConfig) /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true nameSuffix: NativeAOT isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -363,6 +371,7 @@ extends: - windows_x86 # - windows_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:BuildMonoAOTCrossCompiler=false nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -380,6 +389,7 @@ extends: - browser_wasm - wasi_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -396,6 +406,7 @@ extends: platforms: - browser_wasm jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoWasmBuildVariant=multithread /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) nameSuffix: Mono_multithread isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -418,6 +429,7 @@ extends: - ios_arm64 - maccatalyst_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} # @@ -434,6 +446,7 @@ extends: - linux_arm64 - linux_musl_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -458,6 +471,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -483,6 +497,7 @@ extends: - osx_x64 - osx_arm64 jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+packs -c $(_BuildConfig) /p:MonoCrossAOTTargetOS=android+browser+tvos+ios+maccatalyst /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true nameSuffix: CrossAOT_Mono @@ -526,6 +541,7 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=false nameSuffix: Mono_LLVMJIT @@ -540,6 +556,7 @@ extends: buildConfig: release runtimeFlavor: mono jobParameters: + templatePath: 'templates-official' buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) /p:MonoEnableLLVM=true /p:MonoAOTEnableLLVM=true /p:MonoBundleLLVMOptimizer=true nameSuffix: Mono_LLVMAOT @@ -560,6 +577,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' buildArgs: -s tools+libs -allConfigurations -c $(_BuildConfig) /p:TestAssemblies=false /p:TestPackages=true nameSuffix: Libraries_AllConfigurations isOfficialBuild: ${{ variables.isOfficialBuild }} @@ -579,7 +597,9 @@ extends: platforms: - SourceBuild_linux_x64 jobParameters: + templatePath: 'templates-official' nameSuffix: PortableSourceBuild + isOfficialBuild: ${{ variables.isOfficialBuild }} postBuildSteps: - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml parameters: @@ -601,7 +621,8 @@ extends: - windows_arm64 - linux_arm64 jobParameters: - buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument + templatePath: 'templates-official' + buildArgs: -s clr.native+clr.corelib+clr.tools+clr.nativecorelib+libs+host+packs -c $(_BuildConfig) -pgoinstrument /p:SkipLibrariesNativeRuntimePackages=true isOfficialBuild: ${{ variables.isOfficialBuild }} nameSuffix: PGO postBuildSteps: @@ -620,6 +641,7 @@ extends: platforms: - windows_x64 jobParameters: + templatePath: 'templates-official' isOfficialBuild: ${{ variables.isOfficialBuild }} timeoutInMinutes: 120 dependsOn: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index 767b3dc9ba4..8f001c49030 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -52,6 +52,7 @@ variables: extends: template: /eng/pipelines/common/templates/pipeline-with-resources.yml parameters: + isOfficialBuild: false stages: - stage: Build jobs: @@ -155,7 +156,6 @@ extends: - windows_x64 - windows_x86 - windows_arm64 - - freebsd_x64 jobParameters: testGroup: innerloop # Mono/runtimetests also need this, but skip for wasm @@ -187,48 +187,6 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr_jit.containsChange'], true), eq(variables['isRollingBuild'], true))) - # - # Build CoreCLR with no R2R - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - linux_x86 - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NoR2R - buildArgs: -s clr.runtime+clr.jit+clr.iltools+clr.spmi+clr.corelib -c $(_BuildConfig) - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - - # - # Build CoreCLR as a non-portable build - # - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml - buildConfig: checked - runtimeFlavor: coreclr - platforms: - - tizen_armel - jobParameters: - testScope: innerloop - nameSuffix: CoreCLR_NonPortable - buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false - timeoutInMinutes: 120 - condition: >- - or( - eq(dependencies.evaluate_paths.outputs['SetPathVars_coreclr.containsChange'], true), - eq(variables['isRollingBuild'], true)) - # # CoreCLR NativeAOT debug build and smoke tests # Only when CoreCLR is changed @@ -428,6 +386,7 @@ extends: - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/mono/templates/generate-offsets.yml + templatePath: 'templates' buildConfig: release platforms: - android_x64 @@ -1059,7 +1018,6 @@ extends: - osx_arm64 - osx_x64 - windows_x64 - - freebsd_x64 jobParameters: testScope: innerloop condition: @@ -1183,7 +1141,6 @@ extends: - linux_arm64 - linux_musl_x64 - windows_x64 - - freebsd_x64 jobParameters: liveRuntimeBuildConfig: release liveLibrariesBuildConfig: ${{ variables.debugOnPrReleaseOnRolling }} diff --git a/src/coreclr/debug/daccess/CMakeLists.txt b/src/coreclr/debug/daccess/CMakeLists.txt index 9a867c07801..5332e957c9e 100644 --- a/src/coreclr/debug/daccess/CMakeLists.txt +++ b/src/coreclr/debug/daccess/CMakeLists.txt @@ -53,7 +53,7 @@ if(CLR_CMAKE_HOST_FREEBSD OR CLR_CMAKE_HOST_NETBSD OR CLR_CMAKE_HOST_SUNOS) DEPENDS coreclr VERBATIM COMMAND_EXPAND_LISTS - COMMAND ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} + COMMAND ${CMAKE_COMMAND} -E env NM=${CMAKE_NM} ${CLR_DIR}/pal/tools/gen-dactable-rva.sh ${args} COMMENT Generating ${GENERATED_INCLUDE_DIR}/dactablerva.h ) diff --git a/src/coreclr/vm/arm/thunktemplates.S b/src/coreclr/vm/arm/thunktemplates.S index 0686bb2ed4b..8744c8ebb63 100644 --- a/src/coreclr/vm/arm/thunktemplates.S +++ b/src/coreclr/vm/arm/thunktemplates.S @@ -11,7 +11,7 @@ PAGE_SIZE = 4096 -#define DATA_SLOT(stub, field) stub##Code + PAGE_SIZE + stub##Data__##field +#define DATA_SLOT(stub, field) . - (. - stub##Code) + PAGE_SIZE + stub##Data__##field LEAF_ENTRY StubPrecodeCode ldr r12, DATA_SLOT(StubPrecode, MethodDesc) diff --git a/src/mono/mono.proj b/src/mono/mono.proj index 3122cdf153c..a643e5a049e 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -810,6 +810,16 @@ + + + $(ROOTFS_HOST_DIR) + + @@ -859,7 +869,7 @@ <_MonoSkipInitCompiler Condition="'$(CrossBuild)' == 'true'">false <_MonoAotCrossOffsetsCommand Condition="'$(MonoUseCrossTool)' == 'true'">$(PythonCmd) $(MonoProjectRoot)mono/tools/offsets-tool/offsets-tool.py @(MonoAotCrossOffsetsToolParams, ' ') <_MonoAotCMakeConfigureCommand>cmake @(MonoAOTCMakeArgs, ' ') $(MonoCMakeExtraArgs) "$(MonoProjectRoot.TrimEnd('\/'))" - <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' + <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' != 'true' and '$(HostOS)' != 'windows'">sh -c 'build_arch="$(_CompilerTargetArch)" ROOTFS_DIR="$(MonoCrossDir)" compiler="$(MonoCCompiler)" . "$(RepositoryEngineeringCommonDir)native/init-compiler.sh" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand)' <_MonoAotCMakeConfigureCommand Condition="'$(_MonoSkipInitCompiler)' == 'true' and '$(HostOS)' != 'windows'">$(_MonoAOTCCOption) $(_MonoAOTCXXOption) @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeConfigureCommand Condition="'$(HostOS)' == 'windows'">call "$(RepositoryEngineeringDir)native\init-vs-env.cmd" $(_CompilerTargetArch) && cd /D "$(MonoObjCrossDir)" && @(_MonoAotBuildEnv, ' ') $(_MonoAotCMakeConfigureCommand) <_MonoAotCMakeBuildCommand>cmake --build . --target install --config $(Configuration) From 3516014c9efde23f26cc898687c578290e595211 Mon Sep 17 00:00:00 2001 From: Alexey Zakharov Date: Fri, 31 May 2024 01:31:46 +0200 Subject: [PATCH 131/151] Ensure LoaderAllocator can't be collected while we clean handles on collectible LoaderAllocators (#102872) (cherry picked from commit eb84218fd176b5eaef40cf999cc63b6862f36251) --- src/coreclr/vm/threadstatics.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/coreclr/vm/threadstatics.cpp b/src/coreclr/vm/threadstatics.cpp index e3430acf8e4..0216ae054a1 100644 --- a/src/coreclr/vm/threadstatics.cpp +++ b/src/coreclr/vm/threadstatics.cpp @@ -52,13 +52,22 @@ void ThreadLocalBlock::FreeTLM(SIZE_T i, BOOL isThreadShuttingdown) ThreadLocalModule::CollectibleDynamicEntry *entry = (ThreadLocalModule::CollectibleDynamicEntry*)pThreadLocalModule->m_pDynamicClassTable[k].m_pDynamicEntry; PTR_LoaderAllocator pLoaderAllocator = entry->m_pLoaderAllocator; - if (entry->m_hGCStatics != NULL) - { - pLoaderAllocator->FreeHandle(entry->m_hGCStatics); - } - if (entry->m_hNonGCStatics != NULL) + // LoaderAllocator may be collected when the thread is shutting down. + // We enter coop mode to ensure that we get a valid value of the exposed object and + // can safely clean up handles if it is not yet collected. + GCX_COOP(); + + LOADERALLOCATORREF loaderAllocator = pLoaderAllocator->GetExposedObject(); + if (loaderAllocator != NULL) { - pLoaderAllocator->FreeHandle(entry->m_hNonGCStatics); + if (entry->m_hGCStatics != NULL) + { + pLoaderAllocator->FreeHandle(entry->m_hGCStatics); + } + if (entry->m_hNonGCStatics != NULL) + { + pLoaderAllocator->FreeHandle(entry->m_hNonGCStatics); + } } } delete pThreadLocalModule->m_pDynamicClassTable[k].m_pDynamicEntry; From 67133a6768e4750dee143d9eb89fb3baf3e23280 Mon Sep 17 00:00:00 2001 From: Juan Hoyos <19413848+hoyosjs@users.noreply.github.com> Date: Fri, 31 May 2024 11:02:02 -0700 Subject: [PATCH 132/151] Update DIA to 17.10.0-beta1.24272.1 (#102641) --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 26f979c5617..ad4ed3a86fa 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -165,7 +165,7 @@ 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 - 17.8.7-beta1.24113.1 + 17.10.0-beta1.24272.1 2.0.0-beta4.23307.1 3.0.3 2.1.0 From cff625b40a8877beeef603fbded73e91560dc31e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:53:07 -0700 Subject: [PATCH 133/151] Fix a bug in PAL version of _vsnprint_f (#103003) When the formatted string cannot fully fit in the buffer (including its null terminator) `_vsnprint_f` should return -1. However, in the case where the number of chars was the same as the buffer size it was returning the buffer size. Co-authored-by: Jakob Botsch Nielsen --- src/coreclr/pal/src/safecrt/vsprintf.cpp | 2 +- .../palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/coreclr/pal/src/safecrt/vsprintf.cpp b/src/coreclr/pal/src/safecrt/vsprintf.cpp index b8ff745f563..360222d5dc6 100644 --- a/src/coreclr/pal/src/safecrt/vsprintf.cpp +++ b/src/coreclr/pal/src/safecrt/vsprintf.cpp @@ -95,7 +95,7 @@ DLLEXPORT int __cdecl _vsnprintf_s ( retvalue = vsnprintf(string, sizeInBytes, format, ap); string[sizeInBytes - 1] = '\0'; /* we allow truncation if count == _TRUNCATE */ - if (retvalue > (int)sizeInBytes && count == _TRUNCATE) + if (retvalue >= (int)sizeInBytes && count == _TRUNCATE) { if (errno == ERANGE) { diff --git a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp index fb5ab3a2d7a..62b72520876 100644 --- a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp +++ b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp @@ -49,6 +49,18 @@ PALTEST(c_runtime__vsnprintf_s_test1_paltest_vsnprintf_test1, "c_runtime/_vsnpri Fail("ERROR: expected %s (up to %d chars), got %s\n", checkstr, 8, buf); } + char buf8[8] = {0}; + + ret = Testvsnprintf(buf8, 8, "abcdefgh"); + if (ret >= 0) + { + Fail("ERROR: expected negative return value, got %d", ret); + } + if (memcmp(buf8, "abcdefg\0", 8) != 0) + { + Fail("ERROR: Expected 7 chars + null terminator"); + } + PAL_Terminate(); return PASS; } From ca2841138e665611e4ebf4e7e350b673b8268b3b Mon Sep 17 00:00:00 2001 From: Filip Navara Date: Tue, 4 Jun 2024 09:37:51 +0200 Subject: [PATCH 134/151] Fix calculation of channel bindings hash in managed NTLM implementation (#102565) --- .../NegotiateAuthenticationPal.ManagedNtlm.cs | 21 +++++++++++++++---- .../Net/NegotiateAuthenticationPal.Unix.cs | 4 ++-- .../Pal.Managed/SafeChannelBindingHandle.cs | 10 ++++----- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs index 866a754af72..794e2ac5790 100644 --- a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs +++ b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs @@ -416,10 +416,23 @@ private unsafe void WriteChannelBindingHash(Span hashBuffer) { if (_channelBinding != null) { - IntPtr cbtData = _channelBinding.DangerousGetHandle(); - int cbtDataSize = _channelBinding.Size; - int written = MD5.HashData(new Span((void*)cbtData, cbtDataSize), hashBuffer); - Debug.Assert(written == MD5.HashSizeInBytes); + int appDataOffset = sizeof(SecChannelBindings); + IntPtr cbtData = (nint)_channelBinding.DangerousGetHandle() + appDataOffset; + int cbtDataSize = _channelBinding.Size - appDataOffset; + + // Channel bindings are calculated according to RFC 4121, section 4.1.1.2, + // so we need to include zeroed initiator fields and length prefix for the + // application data. + Span prefix = stackalloc byte[sizeof(uint) * 5]; + prefix.Clear(); + BinaryPrimitives.WriteInt32LittleEndian(prefix.Slice(sizeof(uint) * 4), cbtDataSize); + using (var md5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5)) + { + md5.AppendData(prefix); + md5.AppendData(new Span((void*)cbtData, cbtDataSize)); + int written = md5.GetHashAndReset(hashBuffer); + Debug.Assert(written == MD5.HashSizeInBytes); + } } else { diff --git a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs index ed1fe4e2e91..4dba6bcfbd8 100644 --- a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs +++ b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.Unix.cs @@ -551,7 +551,7 @@ Interop.NetSecurityNative.Status status } } - private NegotiateAuthenticationStatusCode InitializeSecurityContext( + private unsafe NegotiateAuthenticationStatusCode InitializeSecurityContext( ref SafeGssCredHandle credentialsHandle, ref SafeGssContextHandle? contextHandle, ref SafeGssNameHandle? targetNameHandle, @@ -594,7 +594,7 @@ private NegotiateAuthenticationStatusCode InitializeSecurityContext( { // If a TLS channel binding token (cbt) is available then get the pointer // to the application specific data. - int appDataOffset = Marshal.SizeOf(); + int appDataOffset = sizeof(SecChannelBindings); Debug.Assert(appDataOffset < channelBinding.Size); IntPtr cbtAppData = channelBinding.DangerousGetHandle() + appDataOffset; int cbtAppDataSize = channelBinding.Size - appDataOffset; diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs b/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs index 35daf739b88..0b8699fd0ea 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/Pal.Managed/SafeChannelBindingHandle.cs @@ -11,7 +11,7 @@ namespace System.Net.Security internal sealed class SafeChannelBindingHandle : ChannelBinding { private const int CertHashMaxSize = 128; - private static readonly int s_secChannelBindingSize = Marshal.SizeOf(); + private static unsafe int SecChannelBindingSize => sizeof(SecChannelBindings); private readonly int _cbtPrefixByteArraySize; internal int Length { get; private set; } @@ -36,8 +36,8 @@ internal unsafe SafeChannelBindingHandle(ChannelBindingKind kind) "tls-unique:"u8; _cbtPrefixByteArraySize = cbtPrefix.Length; - handle = Marshal.AllocHGlobal(s_secChannelBindingSize + _cbtPrefixByteArraySize + CertHashMaxSize); - IntPtr cbtPrefixPtr = handle + s_secChannelBindingSize; + handle = Marshal.AllocHGlobal(SecChannelBindingSize + _cbtPrefixByteArraySize + CertHashMaxSize); + IntPtr cbtPrefixPtr = handle + SecChannelBindingSize; cbtPrefix.CopyTo(new Span((byte*)cbtPrefixPtr, cbtPrefix.Length)); CertHashPtr = cbtPrefixPtr + _cbtPrefixByteArraySize; Length = CertHashMaxSize; @@ -46,12 +46,12 @@ internal unsafe SafeChannelBindingHandle(ChannelBindingKind kind) internal void SetCertHashLength(int certHashLength) { int cbtLength = _cbtPrefixByteArraySize + certHashLength; - Length = s_secChannelBindingSize + cbtLength; + Length = SecChannelBindingSize + cbtLength; SecChannelBindings channelBindings = new SecChannelBindings() { ApplicationDataLength = cbtLength, - ApplicationDataOffset = s_secChannelBindingSize + ApplicationDataOffset = SecChannelBindingSize }; Marshal.StructureToPtr(channelBindings, handle, true); } From bfaf24e2fef15d89408853fbfb92f4bf4a6b6ab8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 4 Jun 2024 09:38:50 +0200 Subject: [PATCH 135/151] [release/8.0] Removed unused sessions from SSL_CTX internal cache (#102095) * Disable OpenSSL internal SSL_SESSION cache for clients * Attempt no. 2 * Revert "Disable OpenSSL internal SSL_SESSION cache for clients" This reverts commit 56a308e88171bb797d13d50953b83262cd8289cd. --------- Co-authored-by: Radek Zikmund --- .../Interop.OpenSsl.cs | 2 +- .../Interop.SslCtx.cs | 36 +++++++++++++------ .../entrypoints.c | 1 + .../opensslshim.h | 2 ++ .../pal_ssl.c | 5 +++ .../pal_ssl.h | 5 +++ 6 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs index c526f37b0b9..110d4714ba5 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.OpenSsl.cs @@ -787,7 +787,7 @@ private static unsafe void RemoveSessionCallback(IntPtr ctx, IntPtr session) IntPtr name = Ssl.SessionGetHostname(session); Debug.Assert(name != IntPtr.Zero); - ctxHandle.RemoveSession(name); + ctxHandle.RemoveSession(name, session); } #if DEBUG diff --git a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs index 5baa776d9d5..d92e15e940e 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.SslCtx.cs @@ -39,6 +39,9 @@ internal static partial class Ssl [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslCtxSetCaching")] internal static unsafe partial int SslCtxSetCaching(SafeSslContextHandle ctx, int mode, int cacheSize, int contextIdLength, Span contextId, delegate* unmanaged neewSessionCallback, delegate* unmanaged removeSessionCallback); + [LibraryImport(Libraries.CryptoNative, EntryPoint = "CryptoNative_SslCtxRemoveSession")] + internal static unsafe partial void SslCtxRemoveSession(SafeSslContextHandle ctx, IntPtr session); + internal static bool AddExtraChainCertificates(SafeSslContextHandle ctx, ReadOnlyCollection chain) { // send pre-computed list of intermediates. @@ -142,27 +145,38 @@ internal bool TryAddSession(IntPtr namePtr, IntPtr session) // This will use strdup() so it is safe to pass in raw pointer. Interop.Ssl.SessionSetHostname(session, namePtr); + IntPtr oldSession = IntPtr.Zero; + lock (_sslSessions) { if (!_sslSessions.TryAdd(targetName, session)) { - if (_sslSessions.Remove(targetName, out IntPtr oldSession)) - { - Interop.Ssl.SessionFree(oldSession); - } - + // session to this target host exists, replace it + _sslSessions.Remove(targetName, out oldSession); bool added = _sslSessions.TryAdd(targetName, session); Debug.Assert(added); } } + if (oldSession != IntPtr.Zero) + { + // remove old session also from the internal OpenSSL cache + // and drop reference count. Since SSL_CTX_remove_session + // will call session_remove_cb, we need to do this outside + // of _sslSessions lock to avoid deadlock with another thread + // which could be holding SSL_CTX lock and trying to acquire + // _sslSessions lock. + Interop.Ssl.SslCtxRemoveSession(this, oldSession); + Interop.Ssl.SessionFree(oldSession); + } + return true; } return false; } - internal void RemoveSession(IntPtr namePtr) + internal void RemoveSession(IntPtr namePtr, IntPtr session) { Debug.Assert(_sslSessions != null); @@ -171,11 +185,14 @@ internal void RemoveSession(IntPtr namePtr) if (_sslSessions != null && targetName != null) { - IntPtr oldSession; - bool removed; + IntPtr oldSession = IntPtr.Zero; + bool removed = false; lock (_sslSessions) { - removed = _sslSessions.Remove(targetName, out oldSession); + if (_sslSessions.TryGetValue(targetName, out IntPtr existingSession) && existingSession == session) + { + removed = _sslSessions.Remove(targetName, out oldSession); + } } if (removed) @@ -209,7 +226,6 @@ internal bool TrySetSession(SafeSslHandle sslHandle, string name) // This will increase reference count on the session as needed. // We need to hold lock here to prevent session being deleted before the call is done. Interop.Ssl.SslSetSession(sslHandle, session); - return true; } } diff --git a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c index f69959f36a4..1045645f171 100644 --- a/src/native/libs/System.Security.Cryptography.Native/entrypoints.c +++ b/src/native/libs/System.Security.Cryptography.Native/entrypoints.c @@ -299,6 +299,7 @@ static const Entry s_cryptoNative[] = DllImportEntry(CryptoNative_IsSslStateOK) DllImportEntry(CryptoNative_SslCtxAddExtraChainCert) DllImportEntry(CryptoNative_SslCtxSetCaching) + DllImportEntry(CryptoNative_SslCtxRemoveSession) DllImportEntry(CryptoNative_SslCtxSetCiphers) DllImportEntry(CryptoNative_SslCtxSetDefaultOcspCallback) DllImportEntry(CryptoNative_SslCtxSetEncryptionPolicy) diff --git a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h index 1f1b1851f09..f94ddc01274 100644 --- a/src/native/libs/System.Security.Cryptography.Native/opensslshim.h +++ b/src/native/libs/System.Security.Cryptography.Native/opensslshim.h @@ -525,6 +525,7 @@ int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, unsigned char *md, size_t len); REQUIRED_FUNCTION(SSL_CTX_new) \ REQUIRED_FUNCTION(SSL_CTX_sess_set_new_cb) \ REQUIRED_FUNCTION(SSL_CTX_sess_set_remove_cb) \ + REQUIRED_FUNCTION(SSL_CTX_remove_session) \ LIGHTUP_FUNCTION(SSL_CTX_set_alpn_protos) \ LIGHTUP_FUNCTION(SSL_CTX_set_alpn_select_cb) \ REQUIRED_FUNCTION(SSL_CTX_set_cipher_list) \ @@ -1040,6 +1041,7 @@ FOR_ALL_OPENSSL_FUNCTIONS #define SSL_CTX_new SSL_CTX_new_ptr #define SSL_CTX_sess_set_new_cb SSL_CTX_sess_set_new_cb_ptr #define SSL_CTX_sess_set_remove_cb SSL_CTX_sess_set_remove_cb_ptr +#define SSL_CTX_remove_session SSL_CTX_remove_session_ptr #define SSL_CTX_set_alpn_protos SSL_CTX_set_alpn_protos_ptr #define SSL_CTX_set_alpn_select_cb SSL_CTX_set_alpn_select_cb_ptr #define SSL_CTX_set_cipher_list SSL_CTX_set_cipher_list_ptr diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c index e6bd41143c1..e320d1c73d7 100644 --- a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c +++ b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.c @@ -701,6 +701,11 @@ int CryptoNative_SslCtxSetCaching(SSL_CTX* ctx, int mode, int cacheSize, int con return retValue; } +int CryptoNative_SslCtxRemoveSession(SSL_CTX* ctx, SSL_SESSION* session) +{ + return SSL_CTX_remove_session(ctx, session); +} + const char* CryptoNative_SslGetServerName(SSL* ssl) { return SSL_get_servername(ssl, TLSEXT_NAMETYPE_host_name); diff --git a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h index 3c63564cc4e..9c9d7026119 100644 --- a/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h +++ b/src/native/libs/System.Security.Cryptography.Native/pal_ssl.h @@ -167,6 +167,11 @@ Sets session caching. 0 is disabled. */ PALEXPORT int CryptoNative_SslCtxSetCaching(SSL_CTX* ctx, int mode, int cacheSize, int contextIdLength, uint8_t* contextId, SslCtxNewSessionCallback newSessionCb, SslCtxRemoveSessionCallback removeSessionCb); +/* +Removes a session from internal cache. +*/ +PALEXPORT int CryptoNative_SslCtxRemoveSession(SSL_CTX* ctx, SSL_SESSION* session); + /* Sets callback to log TLS session keys */ From 5e7120c954c3a0d27ee5ee7658ae5a8898a5ebc4 Mon Sep 17 00:00:00 2001 From: David Mason Date: Tue, 4 Jun 2024 07:28:29 -0700 Subject: [PATCH 136/151] [Release/8.0-staging] Fix issue where the IPC server can fully consume a CPU core and prevent incoming connections (#102530) * 8 fix for ipc * Update ds-ipc-pal-namedpipe.c --- src/native/eventpipe/ds-ipc-pal-namedpipe.c | 25 +++++++++++++++++++++ src/native/eventpipe/ds-ipc-pal-socket.c | 5 +++++ src/native/eventpipe/ds-ipc-pal.h | 3 +++ src/native/eventpipe/ds-ipc.c | 6 ++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/native/eventpipe/ds-ipc-pal-namedpipe.c b/src/native/eventpipe/ds-ipc-pal-namedpipe.c index 01a12275a42..151b3d95cc4 100644 --- a/src/native/eventpipe/ds-ipc-pal-namedpipe.c +++ b/src/native/eventpipe/ds-ipc-pal-namedpipe.c @@ -173,6 +173,27 @@ ds_ipc_free (DiagnosticsIpc *ipc) ep_rt_object_free (ipc); } +void +ds_ipc_reset (DiagnosticsIpc *ipc) +{ + if (!ipc) + return; + + if (ipc->pipe != INVALID_HANDLE_VALUE) { + DisconnectNamedPipe (ipc->pipe); + CloseHandle (ipc->pipe); + ipc->pipe = INVALID_HANDLE_VALUE; + } + + if (ipc->overlap.hEvent != INVALID_HANDLE_VALUE) { + CloseHandle (ipc->overlap.hEvent); + } + + memset(&ipc->overlap, 0, sizeof(OVERLAPPED)); // clear the overlapped objects state + ipc->overlap.hEvent = INVALID_HANDLE_VALUE; + ipc->is_listening = false; +} + int32_t ds_ipc_poll ( DiagnosticsIpcPollHandle *poll_handles_data, @@ -192,6 +213,10 @@ ds_ipc_poll ( // SERVER EP_ASSERT (poll_handles_data [i].ipc->mode == DS_IPC_CONNECTION_MODE_LISTEN); handles [i] = poll_handles_data [i].ipc->overlap.hEvent; + if (handles [i] == INVALID_HANDLE_VALUE) { + // Invalid handle, wait will fail. Signal error + poll_handles_data [i].events = DS_IPC_POLL_EVENTS_ERR; + } } else { // CLIENT bool success = true; diff --git a/src/native/eventpipe/ds-ipc-pal-socket.c b/src/native/eventpipe/ds-ipc-pal-socket.c index d93233c506b..7ad0b0f5d48 100644 --- a/src/native/eventpipe/ds-ipc-pal-socket.c +++ b/src/native/eventpipe/ds-ipc-pal-socket.c @@ -1064,6 +1064,11 @@ ds_ipc_free (DiagnosticsIpc *ipc) ep_rt_object_free (ipc); } +void +ds_ipc_reset (DiagnosticsIpc *ipc) +{ +} + int32_t ds_ipc_poll ( DiagnosticsIpcPollHandle *poll_handles_data, diff --git a/src/native/eventpipe/ds-ipc-pal.h b/src/native/eventpipe/ds-ipc-pal.h index 98e0fba180e..8e246ed6719 100644 --- a/src/native/eventpipe/ds-ipc-pal.h +++ b/src/native/eventpipe/ds-ipc-pal.h @@ -35,6 +35,9 @@ ds_ipc_alloc ( void ds_ipc_free (DiagnosticsIpc *ipc); +void +ds_ipc_reset (DiagnosticsIpc *ipc); + // Poll // Parameters: // - IpcPollHandle * poll_handles_data: Array of IpcPollHandles to poll diff --git a/src/native/eventpipe/ds-ipc.c b/src/native/eventpipe/ds-ipc.c index 1256e3b0033..d3b7292514d 100644 --- a/src/native/eventpipe/ds-ipc.c +++ b/src/native/eventpipe/ds-ipc.c @@ -839,7 +839,11 @@ listen_port_reset ( ds_ipc_error_callback_func callback) { EP_ASSERT (object != NULL); - return; +#ifdef _WIN32 + DiagnosticsListenPort *listen_port = (DiagnosticsListenPort *)object; + ds_ipc_reset (listen_port->port.ipc); + ds_ipc_listen (listen_port->port.ipc, callback); +#endif // _WIN32 } static DiagnosticsPortVtable listen_port_vtable = { From d2f465abe2678c517e64201fa561b8a26c694cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20P=C3=ADchov=C3=A1?= <11718369+ManickaP@users.noreply.github.com> Date: Wed, 5 Jun 2024 08:53:06 +0200 Subject: [PATCH 137/151] [release/8.0-staging] Remove noisy test (#102610) Backport of #100640 to release/8.0-staging /cc @carlossanlop @liveans ## Customer Impact Noisy test, impacts us. ## Regression No ## Testing ## Risk LOW, only test code change --- .../tests/FunctionalTests/MsQuicTests.cs | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs index 51c4279308e..5627009287a 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs @@ -1055,60 +1055,6 @@ public BufferSegment Append(ReadOnlyMemory memory) } } - [Fact] - [OuterLoop("May take several seconds")] - public async Task ByteMixingOrNativeAVE_MinimalFailingTest() - { - const int writeSize = 64 * 1024; - const int NumberOfWrites = 512; - byte[] data1 = new byte[writeSize * NumberOfWrites]; - byte[] data2 = new byte[writeSize * NumberOfWrites]; - Array.Fill(data1, (byte)1); - Array.Fill(data2, (byte)2); - - Task t1 = RunTest(data1); - Task t2 = RunTest(data2); - - async Task RunTest(byte[] data) - { - await RunClientServer( - iterations: 20, - serverFunction: async connection => - { - await using QuicStream stream = await connection.AcceptInboundStreamAsync(); - - byte[] buffer = new byte[data.Length]; - int bytesRead = await ReadAll(stream, buffer); - Assert.Equal(data.Length, bytesRead); - AssertExtensions.SequenceEqual(data, buffer); - - for (int pos = 0; pos < data.Length; pos += writeSize) - { - await stream.WriteAsync(data[pos..(pos + writeSize)]); - } - await stream.WriteAsync(Memory.Empty, completeWrites: true); - }, - clientFunction: async connection => - { - await using QuicStream stream = await connection.OpenOutboundStreamAsync(QuicStreamType.Bidirectional); - - for (int pos = 0; pos < data.Length; pos += writeSize) - { - await stream.WriteAsync(data[pos..(pos + writeSize)]); - } - await stream.WriteAsync(Memory.Empty, completeWrites: true); - - byte[] buffer = new byte[data.Length]; - int bytesRead = await ReadAll(stream, buffer); - Assert.Equal(data.Length, bytesRead); - AssertExtensions.SequenceEqual(data, buffer); - } - ); - } - - await (new[] { t1, t2 }).WhenAllOrAnyFailed(millisecondsTimeout: 1000000); - } - [Fact] public async Task ManagedAVE_MinimalFailingTest() { From b040b7a85fa9159908b9401fc7b4e89d8e18aefb Mon Sep 17 00:00:00 2001 From: Radek Zikmund <32671551+rzikm@users.noreply.github.com> Date: Fri, 7 Jun 2024 07:09:30 +0200 Subject: [PATCH 138/151] [release/8.0] Fix NegotiateStream connections between Linux clients and Windows servers (#102216) * Fix NegotiateStream connections between Linux clients and Windows servers (#99909) * Send the NegotiateSeal NTLM flag when client asked for ProtectionLevel.EncryptAndSign. Process the last handshake done message in NegotiateStream. In case of SPNEGO protocol it may contain message integrity check. Additionally, if the negotiated protocol is NTLM then we need to reset the encryption key after the message integrity check is verified. * Add test for the NegotiateSeal flag * Fix the test * Dummy commit * Fix the new _remoteOk logic in NegotiateStream to fire only when HandshakeComplete. If HandshakeComplete is not true, then the authentication blob will get processed with the normal flow. * Fix the value of NegotiateSeal in the final authentication message of Managed NTLM * Fix build * Remove unwanted test change --------- Co-authored-by: Filip Navara Co-authored-by: wfurt --- .../System/Net/Security/FakeNtlmServer.cs | 14 ++++--- .../NegotiateAuthenticationPal.ManagedNtlm.cs | 21 ++++++++-- .../System/Net/Security/NegotiateStream.cs | 11 +++++- .../UnitTests/NegotiateAuthenticationTests.cs | 38 ++++++++++++++++++- 4 files changed, 72 insertions(+), 12 deletions(-) diff --git a/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs b/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs index cb7a3a785e7..1117b3412f3 100644 --- a/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs +++ b/src/libraries/Common/tests/System/Net/Security/FakeNtlmServer.cs @@ -42,6 +42,8 @@ public FakeNtlmServer(NetworkCredential expectedCredential) public bool IsAuthenticated { get; private set; } public bool IsMICPresent { get; private set; } public string? ClientSpecifiedSpn { get; private set; } + public Flags InitialClientFlags { get; private set; } + public Flags NegotiatedFlags => _negotiatedFlags; private NetworkCredential _expectedCredential; @@ -83,7 +85,7 @@ private enum MessageType : uint } [Flags] - private enum Flags : uint + public enum Flags : uint { NegotiateUnicode = 0x00000001, NegotiateOEM = 0x00000002, @@ -177,17 +179,17 @@ private static ReadOnlySpan GetField(ReadOnlySpan payload, int field case MessageType.Negotiate: // We don't negotiate, we just verify Assert.True(incomingBlob.Length >= 32); - Flags flags = (Flags)BinaryPrimitives.ReadUInt32LittleEndian(incomingBlob.AsSpan(12, 4)); - Assert.Equal(_requiredFlags, (flags & _requiredFlags)); - Assert.True((flags & (Flags.NegotiateOEM | Flags.NegotiateUnicode)) != 0); - if (flags.HasFlag(Flags.NegotiateDomainSupplied)) + InitialClientFlags = (Flags)BinaryPrimitives.ReadUInt32LittleEndian(incomingBlob.AsSpan(12, 4)); + Assert.Equal(_requiredFlags, (InitialClientFlags & _requiredFlags)); + Assert.True((InitialClientFlags & (Flags.NegotiateOEM | Flags.NegotiateUnicode)) != 0); + if (InitialClientFlags.HasFlag(Flags.NegotiateDomainSupplied)) { string domain = Encoding.ASCII.GetString(GetField(incomingBlob, 16)); Assert.Equal(_expectedCredential.Domain, domain); } _expectedMessageType = MessageType.Authenticate; _negotiateMessage = incomingBlob; - return _challengeMessage = GenerateChallenge(flags); + return _challengeMessage = GenerateChallenge(InitialClientFlags); case MessageType.Authenticate: // Validate the authentication! diff --git a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs index 794e2ac5790..00f3369e6ac 100644 --- a/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs +++ b/src/libraries/System.Net.Security/src/System/Net/NegotiateAuthenticationPal.ManagedNtlm.cs @@ -262,8 +262,14 @@ public override void Dispose() { Debug.Assert(incomingBlob.IsEmpty); + Flags requiredFlags = s_requiredFlags; + if (_protectionLevel == ProtectionLevel.EncryptAndSign) + { + requiredFlags |= Flags.NegotiateSeal; + } + _negotiateMessage = new byte[sizeof(NegotiateMessage)]; - CreateNtlmNegotiateMessage(_negotiateMessage); + CreateNtlmNegotiateMessage(_negotiateMessage, requiredFlags); outgoingBlob = _negotiateMessage; statusCode = NegotiateAuthenticationStatusCode.ContinueNeeded; @@ -278,7 +284,7 @@ public override void Dispose() return outgoingBlob; } - private static unsafe void CreateNtlmNegotiateMessage(Span asBytes) + private static unsafe void CreateNtlmNegotiateMessage(Span asBytes, Flags requiredFlags) { Debug.Assert(HeaderLength == NtlmHeader.Length); Debug.Assert(asBytes.Length == sizeof(NegotiateMessage)); @@ -288,7 +294,7 @@ private static unsafe void CreateNtlmNegotiateMessage(Span asBytes) asBytes.Clear(); NtlmHeader.CopyTo(asBytes); message.Header.MessageType = MessageType.Negotiate; - message.Flags = s_requiredFlags; + message.Flags = requiredFlags; message.Version = s_version; } @@ -573,6 +579,13 @@ private static byte[] DeriveKey(ReadOnlySpan exportedSessionKey, ReadOnlyS return null; } + // We already negotiate signing, so we only need to check sealing/encryption. + if ((flags & Flags.NegotiateSeal) == 0 && _protectionLevel == ProtectionLevel.EncryptAndSign) + { + statusCode = NegotiateAuthenticationStatusCode.QopNotSupported; + return null; + } + ReadOnlySpan targetInfo = GetField(challengeMessage.TargetInfo, blob); byte[] targetInfoBuffer = ProcessTargetInfo(targetInfo, out DateTime time, out bool hasNbNames); @@ -607,7 +620,7 @@ private static byte[] DeriveKey(ReadOnlySpan exportedSessionKey, ReadOnlyS NtlmHeader.CopyTo(responseAsSpan); response.Header.MessageType = MessageType.Authenticate; - response.Flags = s_requiredFlags; + response.Flags = s_requiredFlags | (flags & Flags.NegotiateSeal); response.Version = s_version; // Calculate hash for hmac - same for lm2 and ntlm2 diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs b/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs index f26a837e0c1..8026a955635 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/NegotiateStream.cs @@ -883,7 +883,16 @@ private async Task ReceiveBlobAsync(CancellationToken cancellationTo if (_framer.ReadHeader.MessageId == FrameHeader.HandshakeDoneId) { - _remoteOk = true; + if (HandshakeComplete && message.Length > 0) + { + Debug.Assert(_context != null); + _context.GetOutgoingBlob(message, out NegotiateAuthenticationStatusCode statusCode); + _remoteOk = statusCode is NegotiateAuthenticationStatusCode.Completed; + } + else + { + _remoteOk = true; + } } else if (_framer.ReadHeader.MessageId != FrameHeader.HandshakeId) { diff --git a/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs b/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs index ee909c5b091..2bb25b13e6a 100644 --- a/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs +++ b/src/libraries/System.Net.Security/tests/UnitTests/NegotiateAuthenticationTests.cs @@ -190,6 +190,42 @@ public void NtlmIncorrectExchangeTest() Assert.False(fakeNtlmServer.IsAuthenticated); } + [ConditionalFact(nameof(IsNtlmAvailable))] + public void NtlmEncryptionTest() + { + using FakeNtlmServer fakeNtlmServer = new FakeNtlmServer(s_testCredentialRight); + + NegotiateAuthentication ntAuth = new NegotiateAuthentication( + new NegotiateAuthenticationClientOptions + { + Package = "NTLM", + Credential = s_testCredentialRight, + TargetName = "HTTP/foo", + RequiredProtectionLevel = ProtectionLevel.EncryptAndSign + }); + + NegotiateAuthenticationStatusCode statusCode; + byte[]? negotiateBlob = ntAuth.GetOutgoingBlob((byte[])null, out statusCode); + Assert.Equal(NegotiateAuthenticationStatusCode.ContinueNeeded, statusCode); + Assert.NotNull(negotiateBlob); + + byte[]? challengeBlob = fakeNtlmServer.GetOutgoingBlob(negotiateBlob); + Assert.NotNull(challengeBlob); + // Validate that the client sent NegotiateSeal flag + Assert.Equal(FakeNtlmServer.Flags.NegotiateSeal, (fakeNtlmServer.InitialClientFlags & FakeNtlmServer.Flags.NegotiateSeal)); + + byte[]? authenticateBlob = ntAuth.GetOutgoingBlob(challengeBlob, out statusCode); + Assert.Equal(NegotiateAuthenticationStatusCode.Completed, statusCode); + Assert.NotNull(authenticateBlob); + + byte[]? empty = fakeNtlmServer.GetOutgoingBlob(authenticateBlob); + Assert.Null(empty); + Assert.True(fakeNtlmServer.IsAuthenticated); + + // Validate that the NegotiateSeal flag survived the full exchange + Assert.Equal(FakeNtlmServer.Flags.NegotiateSeal, (fakeNtlmServer.NegotiatedFlags & FakeNtlmServer.Flags.NegotiateSeal)); + } + [ConditionalFact(nameof(IsNtlmAvailable))] [ActiveIssue("https://github.com/dotnet/runtime/issues/65678", TestPlatforms.OSX | TestPlatforms.iOS | TestPlatforms.MacCatalyst)] public void NtlmSignatureTest() @@ -218,7 +254,7 @@ public void NtlmSignatureTest() fakeNtlmServer.Unwrap(output.WrittenSpan, temp); Assert.Equal(s_Hello, temp); - // Test creating signature on server side and decoding it with VerifySignature on client side + // Test creating signature on server side and decoding it with VerifySignature on client side byte[] serverSignedMessage = new byte[16 + s_Hello.Length]; fakeNtlmServer.Wrap(s_Hello, serverSignedMessage); output.Clear(); From 188b412d83356f3c6cbda8c5d64db8b428066ed5 Mon Sep 17 00:00:00 2001 From: Jan Vorlicek Date: Mon, 10 Jun 2024 18:37:26 +0200 Subject: [PATCH 139/151] Fix handling exceptions on shutdown (#101915) A recent partial backport of a change to enable propagating exceptions on shutdown was missing two places in the code that were under a different condition than the global exception handling disabling that was removed. I must have made a mistake when testing that change (not setting watson registry settings properly or using a stale build). This change makes it work correctly. --- src/coreclr/vm/excep.cpp | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index b88d6a959bb..9fe9376fd58 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -6696,14 +6696,6 @@ VEH_ACTION WINAPI CLRVectoredExceptionHandlerPhase3(PEXCEPTION_POINTERS pExcepti VEH_ACTION WINAPI CLRVectoredExceptionHandler(PEXCEPTION_POINTERS pExceptionInfo) { - // It is not safe to execute code inside VM after we shutdown EE. One example is DisablePreemptiveGC - // will block forever. - if (g_fForbidEnterEE) - { - return VEH_CONTINUE_SEARCH; - } - - // // DO NOT USE CONTRACTS HERE AS THIS ROUTINE MAY NEVER RETURN. You can use // static contracts, but currently this is all WRAPPER_NO_CONTRACT. @@ -7385,12 +7377,6 @@ LONG WINAPI CLRVectoredExceptionHandlerShim(PEXCEPTION_POINTERS pExceptionInfo) // WARNING WARNING WARNING WARNING WARNING WARNING WARNING // - // If runtime have been disabled, then simply return. - if (g_fForbidEnterEE) - { - return EXCEPTION_CONTINUE_SEARCH; - } - // WARNING // // We must preserve this so that GCStress=4 eh processing doesnt kill last error. From 7b381d47f734f152430a496524bc73858d0d2bbb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:48:47 -0700 Subject: [PATCH 140/151] [release/8.0-staging] Update dependencies from dotnet/runtime-assets (#102494) * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 * Update dependencies from https://github.com/dotnet/runtime-assets build 20240520.1 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.Formats.Tar.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Text.RegularExpressions.TestData , System.Windows.Extensions.TestData From Version 8.0.0-beta.24211.2 -> To Version 8.0.0-beta.24270.1 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 1 - eng/Version.Details.xml | 56 ++++++++++++++++++++--------------------- eng/Versions.props | 28 ++++++++++----------- 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/NuGet.config b/NuGet.config index 3e394d788ef..f6e25f4daf5 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,6 @@ - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3a2af9bbaf8..e619a5f8c0c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -185,57 +185,57 @@ https://github.com/dotnet/arcade 188340e12c0a372b1681ad6a5e72c608021efdba - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 https://github.com/dotnet/llvm-project @@ -358,9 +358,9 @@ https://github.com/dotnet/hotreload-utils 61f137aacabdbd8f279415287a2dd70e150f5eb1 - + https://github.com/dotnet/runtime-assets - dc8ac7417ae48bfbf2516a346723107af0ad603d + 20ef600733c107d19f57de4955dfb025d39b99e3 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index ad4ed3a86fa..a68243225e1 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,20 +143,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 - 8.0.0-beta.24211.2 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 + 8.0.0-beta.24270.1 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 From 95ab6fe36e70b3e576e003977364acee50b69968 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:49:16 -0700 Subject: [PATCH 141/151] [release/8.0] Update dependencies from dotnet/emsdk (#102505) * Update dependencies from https://github.com/dotnet/emsdk build 20240521.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24270.2 -> To Version 8.0.7-servicing.24271.1 * Block v8, same as on net8-staging and net9 branches. * Update dependencies from https://github.com/dotnet/emsdk build 20240527.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24270.2 -> To Version 8.0.7-servicing.24277.3 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Ilona Tomkowicz --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- ...em.Globalization.Calendars.Hybrid.WASM.Tests.csproj | 10 +++++++++- .../System.Globalization.Hybrid.WASM.Tests.csproj | 8 ++++++++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index d74a62c8f12..1bea60ba70c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index aba57a86483..99589f6331a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - a3c7d8205559d673de8b7cdafd6d80d5f4d2cfed + f0463b32eee396d2d1caac58b271d683c7da2333 - + https://github.com/dotnet/emsdk - a3c7d8205559d673de8b7cdafd6d80d5f4d2cfed + f0463b32eee396d2d1caac58b271d683c7da2333 diff --git a/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj b/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj index 5b898363764..79d84271152 100644 --- a/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj +++ b/src/libraries/System.Globalization.Calendars/tests/Hybrid/System.Globalization.Calendars.Hybrid.WASM.Tests.csproj @@ -5,6 +5,14 @@ true true + + + WasmTestOnBrowser + $(TestArchiveRoot)browserornodejs/ + $(TestArchiveTestsRoot)$(OSPlatformConfig)/ + $(DefineConstants);TARGET_BROWSER + true + @@ -106,4 +114,4 @@ - \ No newline at end of file + diff --git a/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj b/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj index a39604c1cfa..f4e811abb49 100644 --- a/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj +++ b/src/libraries/System.Globalization/tests/Hybrid/System.Globalization.Hybrid.WASM.Tests.csproj @@ -6,6 +6,14 @@ true true + + + WasmTestOnBrowser + $(TestArchiveRoot)browserornodejs/ + $(TestArchiveTestsRoot)$(OSPlatformConfig)/ + $(DefineConstants);TARGET_BROWSER + true + From 16715378e5557c4f0fc865e6e3f68785058e65e9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 11:52:01 -0700 Subject: [PATCH 142/151] [release/8.0-staging] Update dependencies from dotnet/emsdk (#102506) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update dependencies from https://github.com/dotnet/emsdk build 20240521.1 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24264.1 -> To Version 8.0.7-servicing.24271.1 * Update dependencies from https://github.com/dotnet/emsdk build 20240527.3 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.6-servicing.24264.1 -> To Version 8.0.7-servicing.24277.3 --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Carlos Sánchez López <1175054+carlossanlop@users.noreply.github.com> --- NuGet.config | 1 + eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index f6e25f4daf5..1bea60ba70c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,6 +9,7 @@ + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index e619a5f8c0c..d2b215265a4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,13 +90,13 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 16d77ddacb12870344abbc4831387cc3e8f97168 + f0463b32eee396d2d1caac58b271d683c7da2333 - + https://github.com/dotnet/emsdk - 16d77ddacb12870344abbc4831387cc3e8f97168 + f0463b32eee396d2d1caac58b271d683c7da2333 diff --git a/eng/Versions.props b/eng/Versions.props index a68243225e1..abc824d58b2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -240,7 +240,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.6 + 8.0.7 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda From 7446a8b922319b88727dce93009fb60a9fed7bc8 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Mon, 10 Jun 2024 12:15:25 -0700 Subject: [PATCH 143/151] [workload] Use the same naming as net9 workload (#103049) * Use the same naming as net9 workload * Add aliases to the versioned packs * Fixup AOTCompiler import * Fix aliases * Update WorkloadManifest.json.in Fix alias * Update WorkloadManifest.targets.in --- ...ad.Mono.Toolchain.Current.Manifest.pkgproj | 1 + .../WorkloadManifest.json.in | 253 +++++++++++------- .../WorkloadManifest.targets.in | 68 ++--- 3 files changed, 196 insertions(+), 126 deletions(-) diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj index e327fcf2635..b7d58cbebf6 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest.pkgproj @@ -42,6 +42,7 @@ + <_WorkloadManifestValues Include="NetVersion" Value="net8" /> <_WorkloadManifestValues Include="WorkloadVersion" Value="$(PackageVersion)" /> <_WorkloadManifestValues Include="PackageVersion" Value="$(PackageVersion)" /> <_WorkloadManifestValues Include="PackageVersionNet6" Value="$(PackageVersionNet6)" /> diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in index 076e642d2b6..a6ca85da25d 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in @@ -5,30 +5,30 @@ }, "workloads": { "wasm-tools": { - "description": ".NET WebAssembly build tools for net8.0", + "description": ".NET WebAssembly build tools for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Sdk", - "Microsoft.NETCore.App.Runtime.Mono.browser-wasm", - "Microsoft.NETCore.App.Runtime.AOT.Cross.browser-wasm" + "Microsoft.NET.Runtime.WebAssembly.Sdk.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.browser-wasm", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.browser-wasm" ], "extends": [ "microsoft-net-runtime-mono-tooling", "microsoft-net-sdk-emscripten" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64"] }, "wasm-experimental": { - "description": ".NET WebAssembly experimental tooling for net8.0", + "description": ".NET WebAssembly experimental tooling for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Templates", - "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm", + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.multithread.${NetVersion}.browser-wasm", ], "extends": [ "wasm-tools" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64" ] }, "wasi-experimental": { - "description": ".NET WASI experimental for net8.0", + "description": ".NET WASI experimental for ${NetVersion}.0", "packs": [ - "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk", - "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm", - "Microsoft.NET.Runtime.WebAssembly.Templates" + "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk.${NetVersion}", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.wasi-wasm", + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "linux-arm64", "osx-x64", "osx-arm64" ] @@ -36,7 +36,7 @@ "mobile-librarybuilder": { "description": "Mobile SDK for building a self-contained .NET native library", "packs": [ - "Microsoft.NET.Runtime.LibraryBuilder.Sdk" + "Microsoft.NET.Runtime.LibraryBuilder.Sdk.${NetVersion}" ], "extends": [ "microsoft-net-runtime-android-aot", "microsoft-net-runtime-ios", "microsoft-net-runtime-maccatalyst", "microsoft-net-runtime-tvos" ], "platforms": [ "win-x64", "win-arm64", "osx-x64", "osx-arm64" ] @@ -45,10 +45,10 @@ "abstract": true, "description": "Android Mono Runtime", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.android-arm", - "Microsoft.NETCore.App.Runtime.Mono.android-arm64", - "Microsoft.NETCore.App.Runtime.Mono.android-x64", - "Microsoft.NETCore.App.Runtime.Mono.android-x86" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x86" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "osx-x64", "osx-arm64" ] @@ -57,10 +57,10 @@ "abstract": true, "description": "Android Mono AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x86", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm", - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x86", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm64" ], "extends": [ "microsoft-net-runtime-android" ], "platforms": [ "win-x64", "win-arm64", "linux-x64", "osx-x64", "osx-arm64" ] @@ -69,9 +69,9 @@ "abstract": true, "description": "iOS Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.ios-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.ios-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-x64" ], "extends": [ "runtimes-ios" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -80,9 +80,9 @@ "abstract": true, "description": "iOS Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.ios-arm64", - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64", - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.ios-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -91,8 +91,8 @@ "abstract": true, "description": "MacCatalyst Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-x64" ], "extends": [ "runtimes-maccatalyst" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -101,8 +101,8 @@ "abstract": true, "description": "MacCatalyst Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64", - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -111,8 +111,8 @@ "abstract": true, "description": "MacOS CoreCLR and Mono Runtime Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.osx-arm64", - "Microsoft.NETCore.App.Runtime.Mono.osx-x64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-x64", "Microsoft.NETCore.App.Runtime.osx-arm64", "Microsoft.NETCore.App.Runtime.osx-x64" ], @@ -123,9 +123,9 @@ "abstract": true, "description": "tvOS Mono Runtime and AOT Workload", "packs": [ - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvos-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-arm64", - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-x64" + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvos-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-arm64", + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-x64" ], "extends": [ "runtimes-tvos" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -134,9 +134,9 @@ "abstract": true, "description": "tvOS Mono Runtime Packs", "packs": [ - "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64", - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64", - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvos-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-arm64", + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-x64" ], "extends": [ "microsoft-net-runtime-mono-tooling" ], "platforms": [ "win-x64", "win-arm64", "osx-arm64", "osx-x64" ] @@ -154,53 +154,83 @@ "abstract": true, "description": "Shared native build tooling for Mono runtime", "packs": [ - "Microsoft.NET.Runtime.MonoAOTCompiler.Task", - "Microsoft.NET.Runtime.MonoTargets.Sdk" + "Microsoft.NET.Runtime.MonoAOTCompiler.Task.${NetVersion}", + "Microsoft.NET.Runtime.MonoTargets.Sdk.${NetVersion}" ] } }, "packs": { - "Microsoft.NET.Runtime.MonoAOTCompiler.Task": { + "Microsoft.NET.Runtime.MonoAOTCompiler.Task.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.MonoAOTCompiler.Task" + } }, - "Microsoft.NET.Runtime.MonoTargets.Sdk": { + "Microsoft.NET.Runtime.MonoTargets.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.MonoTargets.Sdk" + } }, - "Microsoft.NET.Runtime.LibraryBuilder.Sdk": { + "Microsoft.NET.Runtime.LibraryBuilder.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.LibraryBuilder.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Sdk": { + "Microsoft.NET.Runtime.WebAssembly.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk": { + "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk.${NetVersion}": { "kind": "Sdk", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Wasi.Sdk" + } }, - "Microsoft.NET.Runtime.WebAssembly.Templates": { + "Microsoft.NET.Runtime.WebAssembly.Templates.${NetVersion}": { "kind": "template", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NET.Runtime.WebAssembly.Templates" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-arm": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-arm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-arm64": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x64": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-x64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.android-x86": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.android-x86": { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.android-x86" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x86": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x86": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -212,7 +242,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-x86" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -224,7 +254,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -236,7 +266,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-arm" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.android-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.android-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -248,43 +278,64 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.android-arm64" } }, - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-arm64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.maccatalyst-x64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.maccatalyst-x64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.osx-arm64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-arm64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.osx-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.osx-x64": { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.osx-x64": { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.osx-x64" + } }, "Microsoft.NETCore.App.Runtime.osx-arm64": { "kind": "framework", - "version": "${PackageVersion}", + "version": "${PackageVersion}" }, "Microsoft.NETCore.App.Runtime.osx-x64": { "kind": "framework", - "version": "${PackageVersion}", + "version": "${PackageVersion}" }, - "Microsoft.NETCore.App.Runtime.Mono.ios-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.ios-arm64" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.ios-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-arm64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.iossimulator-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.iossimulator-x64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.iossimulator-x64" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvos-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvos-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -292,19 +343,28 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.tvos-arm64", } }, - "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvos-arm64" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvos-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-arm64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-arm64" + } }, - "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.tvossimulator-x64" : { "kind": "framework", "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.tvossimulator-x64" + } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -312,7 +372,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.maccatalyst-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.maccatalyst-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.maccatalyst-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -320,7 +380,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.maccatalyst-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -328,7 +388,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.tvossimulator-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.tvossimulator-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.tvossimulator-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -344,7 +404,7 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.ios-arm64", } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -352,7 +412,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.iossimulator-arm64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.iossimulator-x64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.iossimulator-x64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -360,7 +420,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.iossimulator-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.browser-wasm": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.browser-wasm": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { @@ -372,17 +432,26 @@ "osx-arm64": "Microsoft.NETCore.App.Runtime.AOT.osx-arm64.Cross.browser-wasm" } }, - "Microsoft.NETCore.App.Runtime.Mono.browser-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.browser-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.browser-wasm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.multithread.${NetVersion}.browser-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.multithread.browser-wasm" + } }, - "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm" : { + "Microsoft.NETCore.App.Runtime.Mono.${NetVersion}.wasi-wasm" : { "kind": "framework", - "version": "${PackageVersion}" + "version": "${PackageVersion}", + "alias-to": { + "any": "Microsoft.NETCore.App.Runtime.Mono.wasi-wasm" + } }, "Microsoft.NETCore.App.Runtime.win-x64" : { "kind": "framework", diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in index 70658c5c21a..313514d0b65 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in @@ -82,70 +82,70 @@ - + - - + + - - + + - - - - + + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - - - + + + + + - - - - + + + + From 26e8cc8665a38f932281439f1f2b425af8d08bca Mon Sep 17 00:00:00 2001 From: Andrew Au Date: Mon, 10 Jun 2024 13:03:15 -0700 Subject: [PATCH 144/151] Remove cached cgroup values (#102971) (#103166) --- src/coreclr/gc/env/gcenv.os.h | 2 +- src/coreclr/gc/gc.cpp | 2 +- src/coreclr/gc/unix/gcenv.unix.cpp | 5 ++--- src/coreclr/gc/windows/gcenv.windows.cpp | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/coreclr/gc/env/gcenv.os.h b/src/coreclr/gc/env/gcenv.os.h index c38892b0cad..753b751f798 100644 --- a/src/coreclr/gc/env/gcenv.os.h +++ b/src/coreclr/gc/env/gcenv.os.h @@ -417,7 +417,7 @@ class GCToOSInterface // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. - static uint64_t GetPhysicalMemoryLimit(bool* is_restricted=NULL); + static uint64_t GetPhysicalMemoryLimit(bool* is_restricted=NULL, bool refresh=false); // Get memory status // Parameters: diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 86022c70550..1cf604da8a8 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -52808,7 +52808,7 @@ int gc_heap::refresh_memory_limit() size_t old_heap_hard_limit_poh = heap_hard_limit_oh[poh]; bool old_hard_limit_config_p = hard_limit_config_p; - total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted_physical_mem); + total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted_physical_mem, true); bool succeed = true; diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp index dd591a54426..0dc9790e066 100644 --- a/src/coreclr/gc/unix/gcenv.unix.cpp +++ b/src/coreclr/gc/unix/gcenv.unix.cpp @@ -1121,14 +1121,13 @@ size_t GCToOSInterface::GetVirtualMemoryLimit() // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. -uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted) +uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted, bool refresh) { size_t restricted_limit; if (is_restricted) *is_restricted = false; - // The limit was not cached - if (g_RestrictedPhysicalMemoryLimit == 0) + if (g_RestrictedPhysicalMemoryLimit == 0 || refresh) { restricted_limit = GetRestrictedPhysicalMemoryLimit(); VolatileStore(&g_RestrictedPhysicalMemoryLimit, restricted_limit); diff --git a/src/coreclr/gc/windows/gcenv.windows.cpp b/src/coreclr/gc/windows/gcenv.windows.cpp index f12a64d7ed1..0d2af7904c8 100644 --- a/src/coreclr/gc/windows/gcenv.windows.cpp +++ b/src/coreclr/gc/windows/gcenv.windows.cpp @@ -960,7 +960,7 @@ size_t GCToOSInterface::GetVirtualMemoryLimit() // Remarks: // If a process runs with a restricted memory limit, it returns the limit. If there's no limit // specified, it returns amount of actual physical memory. -uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted) +uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted, bool refresh) { if (is_restricted) *is_restricted = false; From a04bc0c17edc1089544a9b9a6a10cdf696a906fa Mon Sep 17 00:00:00 2001 From: Vlad Brezae Date: Mon, 10 Jun 2024 23:14:14 +0300 Subject: [PATCH 145/151] [mono][interp] Fix type of args when inlining method (#102801) The vars allocated from pushing values on the execution stack might not reflect exactly the actual type of the var. Consider this pattern: condbr BB0 newobj Derived // push var0 of type Derived br BB1 BB0: newobj Base // push var1 of type Base // here we will end up inserting a `mov var1 -> var0` BB1: // top of stack will be seen as being var0 call Because we first reach BB1 with the stack contents of var0, BB1 will end up accessing top of the stack as var0. However the type of var0 at this point is not Derived, since it can also be a Base object. We currently don't update the type of var0, but just update the type information of the top of stack entry when entering BB1. When inlining, after this commit, we use the type information from the stack, rather than the type of the var present on the stack. --- src/mono/mono/mini/interp/transform.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index 0150c3d32e3..bdb2b3f45ea 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -4998,7 +4998,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, arg_locals = (guint32*) g_malloc ((!!signature->hasthis + signature->param_count) * sizeof (guint32)); /* Allocate locals to store inlined method args from stack */ for (int i = signature->param_count - 1; i >= 0; i--) { - MonoType *type = td->locals [td->sp [-1].local].type; + MonoType *type = get_type_from_stack (td->sp [-1].type, td->sp [-1].klass); local = create_interp_local (td, type); arg_locals [i + !!signature->hasthis] = local; store_local (td, local); From 4d4640ffaac85cf1546940355be4a8459f32511f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:32:06 -0700 Subject: [PATCH 146/151] Update dependencies from https://github.com/dotnet/hotreload-utils build 20240521.1 (#102496) Microsoft.DotNet.HotReload.Utils.Generator.BuildTool From Version 8.0.0-alpha.0.24229.2 -> To Version 8.0.0-alpha.0.24271.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/Versions.props | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d2b215265a4..f7b7268e34b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -354,9 +354,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization 67613417f5e1af250e6ddfba79f8f2885d8e90fb - + https://github.com/dotnet/hotreload-utils - 61f137aacabdbd8f279415287a2dd70e150f5eb1 + c804541158619aae93105f54698ca7f149d28232 https://github.com/dotnet/runtime-assets diff --git a/eng/Versions.props b/eng/Versions.props index abc824d58b2..fc841bcad59 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -186,7 +186,7 @@ 8.0.0-prerelease.24229.2 8.0.0-prerelease.24229.2 8.0.0-prerelease.24229.2 - 8.0.0-alpha.0.24229.2 + 8.0.0-alpha.0.24271.1 2.4.2 1.0.0 2.4.5 From 106b02ffe95afaf6e213faf6c15678158274633c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:14:50 -0700 Subject: [PATCH 147/151] Set version in ZIP local header to ZIP64 when file offset is >4GB (#102053) (#103006) * ZipArchiveEntry didn't set ZIP64 in local headers for small files if their offset are > 4GB. * Added System.IO.Compression and System.IO.Packaging tests. --------- Co-authored-by: Gan Keyu --- .../System/IO/Compression/ZipArchiveEntry.cs | 22 +-- .../tests/ZipArchive/zip_LargeFiles.cs | 129 ++++++++++++++---- .../tests/LargeFilesTests.Net.cs | 97 +++++++++++++ .../tests/LargeFilesTests.cs | 78 +++++++++++ .../tests/System.IO.Packaging.Tests.csproj | 4 + .../System.IO.Packaging/tests/Tests.cs | 68 --------- .../tests/XunitAssemblyAttributes.cs | 9 ++ 7 files changed, 306 insertions(+), 101 deletions(-) create mode 100644 src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs create mode 100644 src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs create mode 100644 src/libraries/System.IO.Packaging/tests/XunitAssemblyAttributes.cs diff --git a/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs index 3d415f8e369..7a5ff6136ca 100644 --- a/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs +++ b/src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs @@ -469,7 +469,7 @@ internal void WriteCentralDirectoryFileHeader() bool zip64Needed = false; - if (SizesTooLarge() + if (AreSizesTooLarge #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -490,7 +490,7 @@ internal void WriteCentralDirectoryFileHeader() } - if (_offsetOfLocalHeader > uint.MaxValue + if (IsOffsetTooLarge #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -797,7 +797,11 @@ private bool IsOpenable(bool needToUncompress, bool needToLoadIntoMemory, out st return true; } - private bool SizesTooLarge() => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue; + private bool AreSizesTooLarge => _compressedSize > uint.MaxValue || _uncompressedSize > uint.MaxValue; + + private bool IsOffsetTooLarge => _offsetOfLocalHeader > uint.MaxValue; + + private bool ShouldUseZIP64 => AreSizesTooLarge || IsOffsetTooLarge; // return value is true if we allocated an extra field for 64 bit headers, un/compressed size private bool WriteLocalFileHeader(bool isEmptyFile) @@ -813,6 +817,9 @@ private bool WriteLocalFileHeader(bool isEmptyFile) bool zip64Used = false; uint compressedSizeTruncated, uncompressedSizeTruncated; + // save offset + _offsetOfLocalHeader = writer.BaseStream.Position; + // if we already know that we have an empty file don't worry about anything, just do a straight shot of the header if (isEmptyFile) { @@ -840,7 +847,7 @@ private bool WriteLocalFileHeader(bool isEmptyFile) { // We are in seekable mode so we will not need to write a data descriptor _generalPurposeBitFlag &= ~BitFlagValues.DataDescriptor; - if (SizesTooLarge() + if (ShouldUseZIP64 #if DEBUG_FORCE_ZIP64 || (_archive._forceZip64 && _archive.Mode == ZipArchiveMode.Update) #endif @@ -865,9 +872,6 @@ private bool WriteLocalFileHeader(bool isEmptyFile) } } - // save offset - _offsetOfLocalHeader = writer.BaseStream.Position; - // calculate extra field. if zip64 stuff + original extraField aren't going to fit, dump the original extraField, because this is more important int bigExtraFieldLength = (zip64Used ? zip64ExtraField.TotalSize : 0) + (_lhUnknownExtraFields != null ? ZipGenericExtraField.TotalSize(_lhUnknownExtraFields) : 0); @@ -964,7 +968,7 @@ private void WriteCrcAndSizesInLocalHeader(bool zip64HeaderUsed) long finalPosition = _archive.ArchiveStream.Position; BinaryWriter writer = new BinaryWriter(_archive.ArchiveStream); - bool zip64Needed = SizesTooLarge() + bool zip64Needed = ShouldUseZIP64 #if DEBUG_FORCE_ZIP64 || _archive._forceZip64 #endif @@ -1048,7 +1052,7 @@ private void WriteDataDescriptor() writer.Write(ZipLocalFileHeader.DataDescriptorSignature); writer.Write(_crc32); - if (SizesTooLarge()) + if (AreSizesTooLarge) { writer.Write(_compressedSize); writer.Write(_uncompressedSize); diff --git a/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs b/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs index d240a176b2b..b4623e653af 100644 --- a/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs +++ b/src/libraries/System.IO.Compression/tests/ZipArchive/zip_LargeFiles.cs @@ -1,48 +1,129 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; -using System.Linq; +using System.Reflection; using Xunit; -namespace System.IO.Compression.Tests +namespace System.IO.Compression.Tests; + +[Collection(nameof(DisableParallelization))] +public class zip_LargeFiles : ZipFileTestBase { - [Collection(nameof(DisableParallelization))] - public class zip_LargeFiles : ZipFileTestBase + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [OuterLoop("It requires almost 12 GB of free disk space")] + public static void UnzipOver4GBZipFile() { - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes - [OuterLoop("It requires almost 12 GB of free disk space")] - public static void UnzipOver4GBZipFile() + byte[] buffer = GC.AllocateUninitializedArray(1_000_000_000); // 1 GB + + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); + + try + { + for (byte i = 0; i < 6; i++) + { + File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); + } + + ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); + + using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); + foreach (ZipArchiveEntry entry in zipArchive.Entries) + { + using Stream entryStream = entry.Open(); + + Assert.True(entryStream.CanRead); + Assert.Equal(buffer.Length, entryStream.Length); + } + } + finally { - byte[] buffer = GC.AllocateUninitializedArray(1_000_000_000); // 1 GB + File.Delete(zipArchivePath); + + tempDir.Delete(recursive: true); + } + } + + private static void FillWithHardToCompressData(byte[] buffer) + { + Random.Shared.NextBytes(buffer); + } - string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); - DirectoryInfo tempDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), "over4GB")); + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] + [InlineData(false)] + [InlineData(true)] + public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) + { + // issue #94899 + + CompressionLevel compressLevel = isCompressed ? CompressionLevel.Optimal : CompressionLevel.NoCompression; + byte[] smallBuffer = GC.AllocateUninitializedArray(1000); + byte[] largeBuffer = GC.AllocateUninitializedArray(1_000_000_000); // ~1 GB + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + string LargeFileName = "largefile"; + string SmallFileName = "smallfile"; + uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; + ushort Zip64Version = 45; - try + try + { + using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); + + // Create + using (ZipArchive archive = new(fs, ZipArchiveMode.Create, true)) { - for (byte i = 0; i < 6; i++) + ZipArchiveEntry file = archive.CreateEntry(LargeFileName, compressLevel); + + using (Stream stream = file.Open()) { - File.WriteAllBytes(Path.Combine(tempDir.FullName, $"{i}.test"), buffer); + // Write 5GB of data + for (var i = 0; i < 5; i++) + { + if (isCompressed) + { + FillWithHardToCompressData(largeBuffer); + } + + stream.Write(largeBuffer); + } } - ZipFile.CreateFromDirectory(tempDir.FullName, zipArchivePath, CompressionLevel.NoCompression, includeBaseDirectory: false); + file = archive.CreateEntry(SmallFileName, compressLevel); - using ZipArchive zipArchive = ZipFile.OpenRead(zipArchivePath); - foreach (ZipArchiveEntry entry in zipArchive.Entries) + using (Stream stream = file.Open()) { - using Stream entryStream = entry.Open(); - - Assert.True(entryStream.CanRead); - Assert.Equal(buffer.Length, entryStream.Length); + stream.Write(smallBuffer); } } - finally + + fs.Position = 0; + + // Validate + using (ZipArchive archive = new(fs, ZipArchiveMode.Read)) { - File.Delete(zipArchivePath); + using var reader = new BinaryReader(fs); - tempDir.Delete(recursive: true); + FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); + + if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) + { + Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); + } + + foreach (ZipArchiveEntry entry in archive.Entries) + { + fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; + ushort versionNeeded = reader.ReadUInt16(); + + // Version is not ZIP64 for files with Local Header at >4GB offset. + Assert.Equal(Zip64Version, versionNeeded); + } } } + finally + { + File.Delete(zipArchivePath); + } } } diff --git a/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs new file mode 100644 index 00000000000..c073c09b8f9 --- /dev/null +++ b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.Net.cs @@ -0,0 +1,97 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Compression; +using System.Net.Mime; +using System.Reflection; +using Xunit; + +namespace System.IO.Packaging.Tests; + +public partial class LargeFilesTests +{ + private static void FillWithHardToCompressData(byte[] buffer) + { + Random.Shared.NextBytes(buffer); + } + + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized), nameof(PlatformDetection.Is64BitProcess))] // don't run it on slower runtimes + [InlineData(false)] + [InlineData(true)] + [OuterLoop("It requires 5~6 GB of free disk space and a lot of CPU time for compressed tests")] + public static void CheckZIP64VersionIsSet_ForSmallFilesAfterBigFiles(bool isCompressed) + { + // issue #94899 + + CompressionOption compressionOption = isCompressed ? CompressionOption.Normal : CompressionOption.NotCompressed; + byte[] smallBuffer = GC.AllocateUninitializedArray(1000); + byte[] largeBuffer = GC.AllocateUninitializedArray(1_000_000_000); // ~1 GB + string zipArchivePath = Path.Combine(Path.GetTempPath(), "over4GB.zip"); + Uri largePartUri = PackUriHelper.CreatePartUri(new Uri("large.bin", UriKind.Relative)); + Uri smallPartUri = PackUriHelper.CreatePartUri(new Uri("small.bin", UriKind.Relative)); + uint ZipLocalFileHeader_OffsetToVersionFromHeaderStart = 4; + ushort Zip64Version = 45; + + try + { + using FileStream fs = File.Open(zipArchivePath, FileMode.Create, FileAccess.ReadWrite); + + // Create + using (Package package = Package.Open(fs, FileMode.Create, FileAccess.Write)) + { + PackagePart partLarge = package.CreatePart(largePartUri, MediaTypeNames.Application.Octet, compressionOption); + + using (Stream streamLarge = partLarge.GetStream()) + { + // Write 5GB of data + + for (var i = 0; i < 5; i++) + { + if (isCompressed) + { + FillWithHardToCompressData(largeBuffer); + } + + streamLarge.Write(largeBuffer); + } + } + + PackagePart partSmall = package.CreatePart(smallPartUri, MediaTypeNames.Application.Octet, compressionOption); + + using (Stream streamSmall = partSmall.GetStream()) + { + streamSmall.Write(smallBuffer); + } + } + + + fs.Position = 0; + + // Validate + using (ZipArchive archive = new ZipArchive(fs, ZipArchiveMode.Read)) + { + using var reader = new BinaryReader(fs); + + FieldInfo offsetOfLHField = typeof(ZipArchiveEntry).GetField("_offsetOfLocalHeader", BindingFlags.NonPublic | BindingFlags.Instance); + + if (offsetOfLHField is null || offsetOfLHField.FieldType != typeof(long)) + { + Assert.Fail("Cannot find the private field of _offsetOfLocalHeader in ZipArchiveEntry or the type is not long. Code may be changed after the test is written."); + } + + foreach (ZipArchiveEntry entry in archive.Entries) + { + fs.Position = (long)offsetOfLHField.GetValue(entry) + ZipLocalFileHeader_OffsetToVersionFromHeaderStart; + ushort versionNeeded = reader.ReadUInt16(); + + // Version is not ZIP64 for files with Local Header at >4GB offset. + Assert.Equal(Zip64Version, versionNeeded); + } + } + } + finally + { + File.Delete(zipArchivePath); + } + } +} diff --git a/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs new file mode 100644 index 00000000000..e5e2350d763 --- /dev/null +++ b/src/libraries/System.IO.Packaging/tests/LargeFilesTests.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace System.IO.Packaging.Tests; + +[Collection(nameof(DisableParallelization))] +public partial class LargeFileTests +{ + [Fact] + [OuterLoop] + public void VeryLargePart() + { + // FileAccess.Write is important, this tells ZipPackage to open the underlying ZipArchive in + // ZipArchiveMode.Create mode as opposed to ZipArchiveMode.Update + // When ZipArchive is opened in Create it will write entries directly to the zip stream + // When ZipArchive is opened in Update it will write uncompressed data to memory until + // the archive is closed. + using (Stream stream = new MemoryStream()) + { + Uri partUri = PackUriHelper.CreatePartUri(new Uri("test.bin", UriKind.Relative)); + + // should compress *very well* + byte[] buffer = new byte[1024 * 1024]; + for (int i = 0; i < buffer.Length; i++) + { + buffer[i] = (byte)(i % 2); + } + + const long SizeInMb = 6 * 1024; // 6GB + long totalLength = SizeInMb * buffer.Length; + + // issue on .NET Framework we cannot use FileAccess.Write on a ZipArchive + using (Package package = Package.Open(stream, FileMode.Create, PlatformDetection.IsNetFramework ? FileAccess.ReadWrite : FileAccess.Write)) + { + PackagePart part = package.CreatePart(partUri, + System.Net.Mime.MediaTypeNames.Application.Octet, + CompressionOption.Fast); + + + using (Stream partStream = part.GetStream()) + { + for (long i = 0; i < SizeInMb; i++) + { + partStream.Write(buffer, 0, buffer.Length); + } + } + } + + // reopen for read and make sure we can get the part length & data matches + stream.Seek(0, SeekOrigin.Begin); + using (Package readPackage = Package.Open(stream)) + { + PackagePart part = readPackage.GetPart(partUri); + + using (Stream partStream = part.GetStream()) + { + Assert.Equal(totalLength, partStream.Length); + byte[] readBuffer = new byte[buffer.Length]; + for (long i = 0; i < SizeInMb; i++) + { + int totalRead = 0; + while (totalRead < readBuffer.Length) + { + int actualRead = partStream.Read(readBuffer, totalRead, readBuffer.Length - totalRead); + Assert.InRange(actualRead, 1, readBuffer.Length - totalRead); + totalRead += actualRead; + } + + Assert.Equal(readBuffer.Length, totalRead); + Assert.True(buffer.AsSpan().SequenceEqual(readBuffer)); + } + } + } + } + } +} diff --git a/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj b/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj index 78bc14a789b..c7bdadf60c9 100644 --- a/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj +++ b/src/libraries/System.IO.Packaging/tests/System.IO.Packaging.Tests.csproj @@ -3,10 +3,14 @@ $(NetCoreAppCurrent);$(NetFrameworkCurrent) + + + + diff --git a/src/libraries/System.IO.Packaging/tests/Tests.cs b/src/libraries/System.IO.Packaging/tests/Tests.cs index 4d547db5e1c..c3d8c6d3a39 100644 --- a/src/libraries/System.IO.Packaging/tests/Tests.cs +++ b/src/libraries/System.IO.Packaging/tests/Tests.cs @@ -3771,74 +3771,6 @@ private void ForEachPartWithFileName(Package package, Action Date: Tue, 11 Jun 2024 11:16:51 -0500 Subject: [PATCH 148/151] Fix missing substitution (#103287) --- .../WorkloadManifest.json.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in index a6ca85da25d..b9e4b4539a7 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.json.in @@ -396,7 +396,7 @@ "osx-x64": "Microsoft.NETCore.App.Runtime.AOT.osx-x64.Cross.tvossimulator-x64" } }, - "Microsoft.NETCore.App.Runtime.AOT.Cross.ios-arm64": { + "Microsoft.NETCore.App.Runtime.AOT.Cross.${NetVersion}.ios-arm64": { "kind": "Sdk", "version": "${PackageVersion}", "alias-to": { From 6b7900d82e600f49fee8e6c9c8cc3bee838d9fb4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 10:16:36 -0700 Subject: [PATCH 149/151] Update dependencies from https://github.com/dotnet/emsdk build 20240610.3 (#103291) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24277.3 -> To Version 8.0.7-servicing.24310.3 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 1bea60ba70c..532a7d4a54f 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index da4282cc396..0aec0da8721 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - f0463b32eee396d2d1caac58b271d683c7da2333 + 41fe641713abfc3ed87a017c93f0a649f5172fb1 - + https://github.com/dotnet/emsdk - f0463b32eee396d2d1caac58b271d683c7da2333 + 41fe641713abfc3ed87a017c93f0a649f5172fb1 From 7a45dc80f7b6750a92587f97e8de10d522314420 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 19:58:18 -0700 Subject: [PATCH 150/151] [release/8.0] Update dependencies from dotnet/emsdk (#103388) * Update dependencies from https://github.com/dotnet/emsdk build 20240612.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24310.3 -> To Version 8.0.7-servicing.24312.2 * Update dependencies from https://github.com/dotnet/emsdk build 20240612.2 Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24310.3 -> To Version 8.0.7-servicing.24312.2 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index 532a7d4a54f..d46745fb694 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 0aec0da8721..ac008664d53 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - 41fe641713abfc3ed87a017c93f0a649f5172fb1 + 727397ca6ba44045f3049fdb12d3908da3de8cee - + https://github.com/dotnet/emsdk - 41fe641713abfc3ed87a017c93f0a649f5172fb1 + 727397ca6ba44045f3049fdb12d3908da3de8cee From a82225ee6e7c2f1cc8e064b6320756b547faa358 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 13 Jun 2024 14:01:46 -0500 Subject: [PATCH 151/151] Update dependencies from https://github.com/dotnet/emsdk build 20240613.1 (#103426) Microsoft.SourceBuild.Intermediate.emsdk , Microsoft.NET.Workload.Emscripten.Current.Manifest-8.0.100 From Version 8.0.7-servicing.24312.2 -> To Version 8.0.7-servicing.24313.1 Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 2 +- eng/Version.Details.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NuGet.config b/NuGet.config index d46745fb694..b2329569aa4 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ac008664d53..b3b3b4e6f82 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -92,11 +92,11 @@ https://github.com/dotnet/emsdk - 727397ca6ba44045f3049fdb12d3908da3de8cee + a64772f521c578bc9925578b1384d3a08a02d31d - + https://github.com/dotnet/emsdk - 727397ca6ba44045f3049fdb12d3908da3de8cee + a64772f521c578bc9925578b1384d3a08a02d31d